{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# encoding=utf-8\n",
    "import os.path as osp\n",
    "import os\n",
    "import copy\n",
    "import matplotlib.pyplot as plt\n",
    "import torch\n",
    "from torch.nn import Linear\n",
    "from sklearn.metrics import average_precision_score, roc_auc_score\n",
    "from torch_geometric.data import TemporalData\n",
    "\n",
    "from torch_geometric.nn import TGNMemory, TransformerConv\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch_geometric.nn.models.tgn import (LastNeighborLoader, IdentityMessage, MeanAggregator,\n",
    "                                           LastAggregator)\n",
    "from torch_geometric import *\n",
    "from torch_geometric.utils import negative_sampling\n",
    "\n",
    "from tqdm import tqdm\n",
    "# from .autonotebook import tqdm as notebook_tqdm\n",
    "\n",
    "import networkx as nx\n",
    "import numpy as np\n",
    "import math\n",
    "import copy\n",
    "import re\n",
    "import time\n",
    "import json\n",
    "import pandas as pd\n",
    "from random import choice\n",
    "import gc\n",
    "from graphviz import Digraph\n",
    "import xxhash\n",
    "\n",
    "from datetime import datetime, timezone\n",
    "import time\n",
    "import pytz\n",
    "from time import mktime\n",
    "from datetime import datetime\n",
    "import time\n",
    "\n",
    "\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "\n",
    "\n",
    "def hashgen(l):\n",
    "    \"\"\"Generate a single hash value from a list. @l is a list of\n",
    "    string values, which can be properties of a node/edge. This\n",
    "    function returns a single hashed integer value.\"\"\"\n",
    "    hasher = xxhash.xxh64()\n",
    "    for e in l:\n",
    "        hasher.update(e)\n",
    "    return hasher.intdigest()\n",
    "\n",
    "\n",
    "def datetime_to_ns_time(date):\n",
    "    \"\"\"\n",
    "    :param date: str   format: %Y-%m-%d %H:%M:%S   e.g. 2013-10-10 23:40:00\n",
    "    :return: nano timestamp\n",
    "    \"\"\"\n",
    "    date,ns=date.split('.')\n",
    "\n",
    "    timeArray = time.strptime(date, '%Y-%m-%dT%H:%M:%S')\n",
    "    timeStamp = int(time.mktime(timeArray))\n",
    "    timeStamp = timeStamp * 1000000000\n",
    "    timeStamp += int(ns.split('Z')[0])\n",
    "    return timeStamp\n",
    "\n",
    "\n",
    "def datetime_to_timestamp_US(date):\n",
    "    \"\"\"\n",
    "    :param date: str   format: %Y-%m-%d %H:%M:%S   e.g. 2013-10-10 23:40:00\n",
    "    :return: nano timestamp\n",
    "    \"\"\"\n",
    "    date=date.replace('-04:00','')\n",
    "    if '.' in date:\n",
    "        date,ms=date.split('.')\n",
    "    else:\n",
    "        ms=0\n",
    "    tz = pytz.timezone('Etc/GMT+4')\n",
    "    timeArray = time.strptime(date, \"%Y-%m-%dT%H:%M:%S\")\n",
    "    dt = datetime.fromtimestamp(mktime(timeArray))\n",
    "    timestamp = tz.localize(dt)\n",
    "    timestamp=timestamp.timestamp()\n",
    "    timeStamp = timestamp*1000+int(ms)\n",
    "    return int(timeStamp)\n",
    "\n",
    "\n",
    "def timestamp_to_datetime_US(ns):\n",
    "    \"\"\"\n",
    "    :param date: str   format: %Y-%m-%d %H:%M:%S   e.g. 2013-10-10 23:40:00\n",
    "    :return: nano timestamp\n",
    "    \"\"\"\n",
    "    tz = pytz.timezone('US/Eastern')\n",
    "    ms=ns%1000\n",
    "    ns/=1000\n",
    "    dt = pytz.datetime.datetime.fromtimestamp(int(ns), tz)\n",
    "    s = dt.strftime('%Y-%m-%d %H:%M:%S')\n",
    "    s+='.'+str(ms)\n",
    "#     s += '.' + str(int(int(ns) % 1000000000)).zfill(9)\n",
    "    return s\n",
    "\n",
    "pid_split_symble=\"#_\"\n",
    "\n",
    "host_split_symble=\"_@\"\n",
    "\n",
    "\n",
    "\n",
    "import xxhash\n",
    "\n",
    "def tensor_find(t,x):\n",
    "    t_np=t.numpy()\n",
    "    idx=np.argwhere(t_np==x)\n",
    "    return idx[0][0]+1\n",
    "\n",
    "\n",
    "def std(t):\n",
    "    t = np.array(t)\n",
    "    return np.std(t)\n",
    "\n",
    "\n",
    "def var(t):\n",
    "    t = np.array(t)\n",
    "    return np.var(t)\n",
    "\n",
    "\n",
    "def mean(t):\n",
    "    t = np.array(t)\n",
    "    return np.mean(t)\n",
    "\n",
    "def hashgen(l):\n",
    "    \"\"\"Generate a single hash value from a list. @l is a list of\n",
    "    string values, which can be properties of a node/edge. This\n",
    "    function returns a single hashed integer value.\"\"\"\n",
    "    hasher = xxhash.xxh64()\n",
    "    for e in l:\n",
    "        hasher.update(e)\n",
    "    return hasher.intdigest()\n",
    "\n",
    "\n",
    "def cal_pos_edges_loss(link_pred_ratio):\n",
    "    loss=[]\n",
    "    for i in link_pred_ratio:\n",
    "        loss.append(criterion(i,torch.ones(1)))\n",
    "    return torch.tensor(loss)\n",
    "\n",
    "def cal_pos_edges_loss_multiclass(link_pred_ratio,labels):\n",
    "    loss=[] \n",
    "    for i in range(len(link_pred_ratio)):\n",
    "        loss.append(criterion(link_pred_ratio[i].reshape(1,-1),labels[i].reshape(-1)))\n",
    "    return torch.tensor(loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Connect to database"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import psycopg2\n",
    "\n",
    "from psycopg2 import extras as ex\n",
    "connect = psycopg2.connect(database = 'optc_db',\n",
    "                           host = '/var/run/postgresql/',\n",
    "                           user = 'postgres',\n",
    "                           password = 'postgres',\n",
    "                           port = '5432'\n",
    "                          )\n",
    "\n",
    "cur = connect.cursor()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load data"
   ]
  },
  {
   "cell_type": "raw",
   "source": [
    "training data"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "graph_9_22_h201=torch.load(\"./data/evaluation/9_22_host=SysClient0201_datalabel=benign.TemporalData\")\n",
    "graph_9_22_h402=torch.load(\"./data/evaluation/9_22_host=SysClient0402_datalabel=benign.TemporalData\")\n",
    "graph_9_22_h660=torch.load(\"./data/evaluation/9_22_host=SysClient0660_datalabel=benign.TemporalData\")\n",
    "graph_9_22_h501=torch.load(\"./data/evaluation/9_22_host=SysClient0501_datalabel=benign.TemporalData\")\n",
    "graph_9_22_h051=torch.load(\"./data/evaluation/9_22_host=SysClient0051_datalabel=benign.TemporalData\")\n",
    "graph_9_22_h209=torch.load(\"./data/evaluation/9_22_host=SysClient0209_datalabel=benign.TemporalData\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data=graph_9_22_h660"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Testing data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# \n",
    "\n",
    "graph_9_23_h201=torch.load(\"./data/evaluation/9_23_host=SysClient0201_datalabel=evaluation.TemporalData\")\n",
    "graph_9_23_h402=torch.load(\"./data/evaluation/9_23_host=SysClient0402_datalabel=evaluation.TemporalData\")\n",
    "graph_9_23_h660=torch.load(\"./data/evaluation/9_23_host=SysClient0660_datalabel=evaluation.TemporalData\")\n",
    "graph_9_23_h501=torch.load(\"./data/evaluation/9_23_host=SysClient0501_datalabel=evaluation.TemporalData\")\n",
    "graph_9_23_h051=torch.load(\"./data/evaluation/9_23_host=SysClient0051_datalabel=evaluation.TemporalData\")\n",
    "graph_9_23_h207=torch.load(\"./data/evaluation/9_23_host=SysClient0207_datalabel=evaluation.TemporalData\")\n",
    "\n",
    "\n",
    "graph_9_24_h201=torch.load(\"./data/evaluation/9_24_host=SysClient0201_datalabel=evaluation.TemporalData\")\n",
    "graph_9_24_h402=torch.load(\"./data/evaluation/9_24_host=SysClient0402_datalabel=evaluation.TemporalData\")\n",
    "graph_9_24_h660=torch.load(\"./data/evaluation/9_24_host=SysClient0660_datalabel=evaluation.TemporalData\")\n",
    "graph_9_24_h501=torch.load(\"./data/evaluation/9_24_host=SysClient0501_datalabel=evaluation.TemporalData\")\n",
    "graph_9_24_h051=torch.load(\"./data/evaluation/9_24_host=SysClient0051_datalabel=evaluation.TemporalData\")\n",
    "graph_9_24_h207=torch.load(\"./data/evaluation/9_24_host=SysClient0207_datalabel=evaluation.TemporalData\")\n",
    "\n",
    "\n",
    "graph_9_25_h201=torch.load(\"./data/evaluation/9_25_host=SysClient0201_datalabel=evaluation.TemporalData\")\n",
    "graph_9_25_h402=torch.load(\"./data/evaluation/9_25_host=SysClient0402_datalabel=evaluation.TemporalData\")\n",
    "graph_9_25_h660=torch.load(\"./data/evaluation/9_25_host=SysClient0660_datalabel=evaluation.TemporalData\")\n",
    "graph_9_25_h501=torch.load(\"./data/evaluation/9_25_host=SysClient0501_datalabel=evaluation.TemporalData\")\n",
    "graph_9_25_h051=torch.load(\"./data/evaluation/9_25_host=SysClient0051_datalabel=evaluation.TemporalData\")\n",
    "graph_9_25_h207=torch.load(\"./data/evaluation/9_25_host=SysClient0207_datalabel=evaluation.TemporalData\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data=graph_9_25_h207"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "graph_9_19=torch.load(\"./data/benign/9-19-h201.TemporalData\")\n",
    "graph_9_20=torch.load(\"./data/benign/9-20-h201.TemporalData\")\n",
    "graph_9_21=torch.load(\"./data/benign/9-21-h201.TemporalData\")\n",
    "graph_9_22=torch.load(\"./data/benign/9-22-h201.TemporalData\")\n",
    "graph_9_23=torch.load(\"./data/evaluation/9-23-h201.TemporalData\")\n",
    "\n",
    "train_data=graph_9_22"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generate node2msg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Constructing the map for nodeid to msg\n",
    "sql=\"select * from nodeid2msg;\"\n",
    "cur.execute(sql)\n",
    "rows = cur.fetchall()\n",
    "\n",
    "node_uuid2path={}  # nodeid => msg and node hash => nodeid\n",
    "for i in tqdm(rows):\n",
    "    node_uuid2path[i[0]]=i[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The node2index of training data\n",
    "node_uuid2index_9_17_h201=torch.load(\"node_uuid2index_9_17_host=SysClient0201_datalabel=benign\")\n",
    "node_uuid2index_9_18_h201=torch.load(\"node_uuid2index_9_18_host=SysClient0201_datalabel=benign\")\n",
    "node_uuid2index_9_19_h201=torch.load(\"node_uuid2index_9_19_host=SysClient0201_datalabel=benign\")\n",
    "node_uuid2index_9_20_h201=torch.load(\"node_uuid2index_9_20_host=SysClient0201_datalabel=benign\")\n",
    "node_uuid2index_9_21_h201=torch.load(\"node_uuid2index_9_21_host=SysClient0201_datalabel=benign\")\n",
    "node_uuid2index_9_22_h201=torch.load(\"node_uuid2index_9_22_host=SysClient0201_datalabel=benign\")\n",
    "node_uuid2index_9_23_h201=torch.load(\"node_uuid2index_9_23_host=SysClient0201_datalabel=benign\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "node_uuid2index_9_22_h201=torch.load(\"node_uuid2index_9_22_host=SysClient0201_datalabel=benign\")\n",
    "node_uuid2index_9_22_h402=torch.load(\"node_uuid2index_9_22_host=SysClient0402_datalabel=benign\")\n",
    "node_uuid2index_9_22_h660=torch.load(\"node_uuid2index_9_22_host=SysClient0660_datalabel=benign\")\n",
    "node_uuid2index_9_22_h501=torch.load(\"node_uuid2index_9_22_host=SysClient0501_datalabel=benign\")\n",
    "node_uuid2index_9_22_h051=torch.load(\"node_uuid2index_9_22_host=SysClient0051_datalabel=benign\")\n",
    "node_uuid2index_9_22_h209=torch.load(\"node_uuid2index_9_22_host=SysClient0209_datalabel=benign\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The node2index of testing data\n",
    "node_uuid2index_9_23_h201=torch.load(\"node_uuid2index_9_23_host=SysClient0201_datalabel=evaluation\")\n",
    "node_uuid2index_9_24_h201=torch.load(\"node_uuid2index_9_24_host=SysClient0201_datalabel=evaluation\")\n",
    "node_uuid2index_9_25_h201=torch.load(\"node_uuid2index_9_25_host=SysClient0201_datalabel=evaluation\")\n",
    "\n",
    "node_uuid2index_9_23_h402=torch.load(\"node_uuid2index_9_23_host=SysClient0402_datalabel=evaluation\")\n",
    "node_uuid2index_9_24_h402=torch.load(\"node_uuid2index_9_24_host=SysClient0402_datalabel=evaluation\")\n",
    "node_uuid2index_9_25_h402=torch.load(\"node_uuid2index_9_25_host=SysClient0402_datalabel=evaluation\")\n",
    "\n",
    "node_uuid2index_9_23_h660=torch.load(\"node_uuid2index_9_23_host=SysClient0660_datalabel=evaluation\")\n",
    "node_uuid2index_9_24_h660=torch.load(\"node_uuid2index_9_24_host=SysClient0660_datalabel=evaluation\")\n",
    "node_uuid2index_9_25_h660=torch.load(\"node_uuid2index_9_25_host=SysClient0660_datalabel=evaluation\")\n",
    "\n",
    "node_uuid2index_9_23_h501=torch.load(\"node_uuid2index_9_23_host=SysClient0501_datalabel=evaluation\")\n",
    "node_uuid2index_9_24_h501=torch.load(\"node_uuid2index_9_24_host=SysClient0501_datalabel=evaluation\")\n",
    "node_uuid2index_9_25_h501=torch.load(\"node_uuid2index_9_25_host=SysClient0501_datalabel=evaluation\")\n",
    "\n",
    "node_uuid2index_9_23_h051=torch.load(\"node_uuid2index_9_23_host=SysClient0051_datalabel=evaluation\")\n",
    "node_uuid2index_9_24_h051=torch.load(\"node_uuid2index_9_24_host=SysClient0051_datalabel=evaluation\")\n",
    "node_uuid2index_9_25_h051=torch.load(\"node_uuid2index_9_25_host=SysClient0051_datalabel=evaluation\")\n",
    "\n",
    "node_uuid2index_9_23_h207=torch.load(\"node_uuid2index_9_23_host=SysClient0207_datalabel=evaluation\")\n",
    "node_uuid2index_9_24_h207=torch.load(\"node_uuid2index_9_24_host=SysClient0207_datalabel=evaluation\")\n",
    "node_uuid2index_9_25_h207=torch.load(\"node_uuid2index_9_25_host=SysClient0207_datalabel=evaluation\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "maxnode_num=max(\n",
    "    len(node_uuid2index_9_17_h201)//2+1,\n",
    "    len(node_uuid2index_9_18_h201)//2+1,\n",
    "    len(node_uuid2index_9_19_h201)//2+1,\n",
    "    len(node_uuid2index_9_20_h201)//2+1,\n",
    "    len(node_uuid2index_9_21_h201)//2+1,\n",
    "    len(node_uuid2index_9_22_h201)//2+1,\n",
    "    len(node_uuid2index_9_23_h201)//2+1,\n",
    "    \n",
    "    len(node_uuid2index_9_22_h201)//2+1,\n",
    "    len(node_uuid2index_9_22_h402)//2+1,\n",
    "    len(node_uuid2index_9_22_h660)//2+1,\n",
    "    len(node_uuid2index_9_22_h501)//2+1,\n",
    "    len(node_uuid2index_9_22_h051)//2+1,\n",
    "    len(node_uuid2index_9_22_h209)//2+1,\n",
    "    \n",
    "    len(node_uuid2index_9_23_h201)//2+1,\n",
    "    len(node_uuid2index_9_24_h201)//2+1,\n",
    "    len(node_uuid2index_9_25_h201)//2+1,\n",
    "    len(node_uuid2index_9_23_h402)//2+1,\n",
    "    len(node_uuid2index_9_24_h402)//2+1,\n",
    "    len(node_uuid2index_9_25_h402)//2+1,    \n",
    "    \n",
    "    len(node_uuid2index_9_23_h660)//2+1,\n",
    "    len(node_uuid2index_9_24_h660)//2+1,\n",
    "    len(node_uuid2index_9_25_h660)//2+1,\n",
    "    \n",
    "    len(node_uuid2index_9_23_h501)//2+1,\n",
    "    len(node_uuid2index_9_24_h501)//2+1,\n",
    "    len(node_uuid2index_9_25_h501)//2+1,\n",
    "    \n",
    "    len(node_uuid2index_9_23_h051)//2+1,\n",
    "    len(node_uuid2index_9_24_h051)//2+1,\n",
    "    len(node_uuid2index_9_25_h051)//2+1,\n",
    "    \n",
    "    len(node_uuid2index_9_23_h207)//2+1,\n",
    "    len(node_uuid2index_9_24_h207)//2+1,\n",
    "    len(node_uuid2index_9_25_h207)//2+1,    \n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "maxnode_num"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rel2id={1: 'OPEN',\n",
    " 'OPEN': 1,\n",
    " 2: 'READ',\n",
    " 'READ': 2,\n",
    " 3: 'CREATE',\n",
    " 'CREATE': 3,\n",
    " 4: 'MESSAGE',\n",
    " 'MESSAGE': 4,\n",
    " 5: 'MODIFY',\n",
    " 'MODIFY': 5,\n",
    " 6: 'START',\n",
    " 'START': 6,\n",
    " 7: 'RENAME',\n",
    " 'RENAME': 7,\n",
    " 8: 'DELETE',\n",
    " 'DELETE': 8,\n",
    " 9: 'TERMINATE',\n",
    " 'TERMINATE': 9,\n",
    " 10: 'WRITE',\n",
    " 'WRITE': 10}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Setting the parameters and Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "time_dim = 100\n",
    "edge_embedding_dimension=100\n",
    "embedding_dim = edge_embedding_dimension\n",
    "neighbor_size=20\n",
    "memory_dim = 100\n",
    "\n",
    "max_node_num = maxnode_num+2\n",
    "min_dst_idx, max_dst_idx = 0, max_node_num\n",
    "neighbor_loader = LastNeighborLoader(max_node_num, size=neighbor_size, device=device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "BATCH=1024\n",
    "class GraphAttentionEmbedding(torch.nn.Module):\n",
    "    def __init__(self, in_channels, out_channels, msg_dim, time_enc):\n",
    "        super(GraphAttentionEmbedding, self).__init__()\n",
    "        self.time_enc = time_enc\n",
    "        edge_dim = msg_dim + time_enc.out_channels\n",
    "        self.conv = TransformerConv(in_channels, out_channels, heads=8,\n",
    "                                    dropout=0.0, edge_dim=edge_dim)\n",
    "        self.conv2 = TransformerConv(out_channels*8, out_channels,heads=1, concat=False,\n",
    "                             dropout=0.0, edge_dim=edge_dim)\n",
    "\n",
    "    def forward(self, x, last_update, edge_index, t, msg):\n",
    "        last_update.to(device)\n",
    "        x = x.to(device)\n",
    "        t = t.to(device)\n",
    "        rel_t = last_update[edge_index[0]] - t\n",
    "        rel_t_enc = self.time_enc(rel_t.to(x.dtype))\n",
    "        edge_attr = torch.cat([rel_t_enc, msg], dim=-1)\n",
    "        x = F.relu(self.conv(x, edge_index, edge_attr))\n",
    "        x = F.relu(self.conv2(x, edge_index, edge_attr))\n",
    "        return x\n",
    "\n",
    "\n",
    "class LinkPredictor(torch.nn.Module):\n",
    "    def __init__(self, in_channels):\n",
    "        super(LinkPredictor, self).__init__()\n",
    "        self.lin_src = Linear(in_channels, in_channels*2)\n",
    "        self.lin_dst = Linear(in_channels, in_channels*2)\n",
    "        self.lin_seq = nn.Sequential(\n",
    "            Linear(in_channels * 4, in_channels * 8),\n",
    "            torch.nn.BatchNorm1d(in_channels * 8),\n",
    "            torch.nn.Dropout(0.5),\n",
    "            nn.Tanh(),\n",
    "            Linear(in_channels * 8, in_channels * 2),\n",
    "            torch.nn.BatchNorm1d(in_channels * 2),\n",
    "            torch.nn.Dropout(0.5),\n",
    "            nn.Tanh(),\n",
    "            Linear(in_channels * 2, int(in_channels // 2)),\n",
    "            torch.nn.BatchNorm1d(int(in_channels // 2)),\n",
    "            torch.nn.Dropout(0.5),\n",
    "            nn.Tanh(),\n",
    "            Linear(int(in_channels // 2), train_data.msg.shape[1] - 32)\n",
    "        )\n",
    "\n",
    "    def forward(self, z_src, z_dst):\n",
    "        h = torch.cat([self.lin_src(z_src) , self.lin_dst(z_dst)],dim=-1)      \n",
    "        h = self.lin_seq (h)        \n",
    "        return h\n",
    "\n",
    "memory = TGNMemory(\n",
    "    max_node_num,\n",
    "    train_data.msg.size(-1),\n",
    "    memory_dim,\n",
    "    time_dim,\n",
    "    message_module=IdentityMessage(train_data.msg.size(-1), memory_dim, time_dim),\n",
    "    aggregator_module=LastAggregator(),\n",
    ").to(device)\n",
    "\n",
    "gnn = GraphAttentionEmbedding(\n",
    "    in_channels=memory_dim,\n",
    "    out_channels=embedding_dim,\n",
    "    msg_dim=train_data.msg.size(-1),\n",
    "    time_enc=memory.time_enc,\n",
    ").to(device)\n",
    "\n",
    "link_pred = LinkPredictor(in_channels=embedding_dim).to(device)\n",
    "\n",
    "optimizer = torch.optim.Adam(\n",
    "    set(memory.parameters()) | set(gnn.parameters())\n",
    "    | set(link_pred.parameters()), lr=0.00005, eps=1e-08,weight_decay=0.01)\n",
    "\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "\n",
    "\n",
    "assoc = torch.empty(max_node_num, dtype=torch.long, device=device)\n",
    "\n",
    "\n",
    "saved_nodes=set()\n",
    "\n",
    "BATCH=1024\n",
    "def train(train_data):\n",
    "\n",
    "\n",
    "    memory.train()\n",
    "    gnn.train()\n",
    "    link_pred.train()\n",
    "\n",
    "    memory.reset_state()  # Start with a fresh memory.\n",
    "    neighbor_loader.reset_state()  # Start with an empty graph.\n",
    "    saved_nodes=set()\n",
    "\n",
    "    total_loss = 0\n",
    "\n",
    "    for batch in train_data.seq_batches(batch_size=BATCH):\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        src, pos_dst, t, msg = batch.src, batch.dst, batch.t, batch.msg        \n",
    "\n",
    "        n_id = torch.cat([src, pos_dst]).unique()\n",
    "\n",
    "        n_id, edge_index, e_id = neighbor_loader(n_id)\n",
    "        assoc[n_id] = torch.arange(n_id.size(0), device=device)\n",
    "\n",
    "        # Get updated memory of all nodes involved in the computation.\n",
    "        z, last_update = memory(n_id)\n",
    "\n",
    "        z = gnn(z, last_update, edge_index, train_data.t[e_id], train_data.msg[e_id])\n",
    "\n",
    "        pos_out = link_pred(z[assoc[src]], z[assoc[pos_dst]])       \n",
    "\n",
    "        y_pred = torch.cat([pos_out], dim=0)\n",
    "\n",
    "        y_true=[]\n",
    "        for m in msg:\n",
    "            l=tensor_find(m[16:-16],1)-1\n",
    "            y_true.append(l)           \n",
    "\n",
    "        y_true = torch.tensor(y_true)\n",
    "        y_true=y_true.reshape(-1).to(torch.long)\n",
    "\n",
    "        loss = criterion(y_pred, y_true)     \n",
    "\n",
    "        # Update memory and neighbor loader with ground-truth state.\n",
    "        memory.update_state(src, pos_dst, t, msg)\n",
    "        neighbor_loader.insert(src, pos_dst)       \n",
    "\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        memory.detach()\n",
    "        total_loss += float(loss) * batch.num_events\n",
    "    return total_loss / train_data.num_events\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_graphs=[\n",
    "    graph_9_22_h201,\n",
    "    graph_9_22_h402,\n",
    "    graph_9_22_h660,\n",
    "    graph_9_22_h501,\n",
    "    graph_9_22_h051,\n",
    "    graph_9_22_h209,\n",
    "]\n",
    "print(f\"{embedding_dim=}\")\n",
    "print(f\"{gnn=}\")\n",
    "for epoch in tqdm(range(1, 11)):\n",
    "    for g in train_graphs:\n",
    "        loss = train(g)\n",
    "        print(f'  Epoch: {epoch:02d}, Loss: {loss:.4f}')\n",
    "\n",
    "\n",
    "memory.reset_state()  # Start with a fresxh memory.\n",
    "neighbor_loader.reset_state() \n",
    "model=[memory,gnn, link_pred,neighbor_loader]\n",
    "torch.save(model,f\"./models/model_saved_traindata=hosts_9_22.pt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define the function for testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time \n",
    "\n",
    "@torch.no_grad()\n",
    "def test_day_new(inference_data,path,nodeuuid2index):\n",
    "    if os.path.exists(path):\n",
    "        pass\n",
    "    else:\n",
    "        os.mkdir(path)\n",
    "    \n",
    "    memory.eval()\n",
    "    gnn.eval()\n",
    "    link_pred.eval()\n",
    "    \n",
    "    memory.reset_state()  \n",
    "    neighbor_loader.reset_state()  \n",
    "    \n",
    "    time_with_loss={}\n",
    "    total_loss = 0    \n",
    "    edge_list=[]\n",
    "    \n",
    "    unique_nodes=torch.tensor([])\n",
    "    total_edges=0\n",
    "    \n",
    "\n",
    "\n",
    "    start_time=int(inference_data.t[0])\n",
    "    event_count=0\n",
    "    \n",
    "    pos_o=[]\n",
    "    \n",
    "    loss_list=[]\n",
    "\n",
    "\n",
    "    print(\"after merge:\",inference_data)\n",
    "    \n",
    "\n",
    "    start = time.perf_counter()\n",
    "\n",
    "    for batch in inference_data.seq_batches(batch_size=BATCH):\n",
    "        \n",
    "        src, pos_dst, t, msg = batch.src, batch.dst, batch.t, batch.msg\n",
    "        unique_nodes=torch.cat([unique_nodes,src,pos_dst]).unique()\n",
    "        total_edges+=BATCH\n",
    "        \n",
    "       \n",
    "        n_id = torch.cat([src, pos_dst]).unique()       \n",
    "        n_id, edge_index, e_id = neighbor_loader(n_id)\n",
    "        assoc[n_id] = torch.arange(n_id.size(0), device=device)\n",
    "\n",
    "        z, last_update = memory(n_id)\n",
    "       \n",
    "        z = gnn(z, last_update, edge_index, inference_data.t[e_id], inference_data.msg[e_id])\n",
    "\n",
    "        pos_out = link_pred(z[assoc[src]], z[assoc[pos_dst]])\n",
    "        \n",
    "        pos_o.append(pos_out)\n",
    "        y_pred = torch.cat([pos_out], dim=0)\n",
    "\n",
    "        y_true=[]\n",
    "        for m in msg:\n",
    "            l=tensor_find(m[16:-16],1)-1\n",
    "            y_true.append(l) \n",
    "        y_true = torch.tensor(y_true)\n",
    "        y_true=y_true.reshape(-1).to(torch.long)\n",
    "\n",
    "        loss = criterion(y_pred, y_true)\n",
    "\n",
    "        total_loss += float(loss) * batch.num_events\n",
    "     \n",
    "        memory.update_state(src, pos_dst, t, msg)\n",
    "        neighbor_loader.insert(src, pos_dst)\n",
    "        \n",
    "\n",
    "        each_edge_loss= cal_pos_edges_loss_multiclass(pos_out,y_true)\n",
    "        \n",
    "        for i in range(len(pos_out)):\n",
    "            srcnode= int(src[i])\n",
    "            dstnode=  int(pos_dst[i])\n",
    "            \n",
    "            srcmsg=str(nodeuuid2index[srcnode]) \n",
    "            dstmsg=str(nodeuuid2index[dstnode])\n",
    "            t_var=int(t[i])\n",
    "            edgeindex=tensor_find(msg[i][16:-16],1) \n",
    "            edge_type=rel2id[edgeindex]\n",
    "            loss=each_edge_loss[i]    \n",
    "\n",
    "            temp_dic={}\n",
    "            temp_dic['loss']=float(loss)\n",
    "            temp_dic['srcnode']=srcnode\n",
    "            temp_dic['dstnode']=dstnode\n",
    "            temp_dic['srcmsg']=srcmsg\n",
    "            temp_dic['dstmsg']=dstmsg\n",
    "            temp_dic['edge_type']=edge_type\n",
    "            temp_dic['time']=t_var\n",
    "\n",
    "            edge_list.append(temp_dic)\n",
    "        \n",
    "        event_count+=len(batch.src)\n",
    "        if t[-1]>start_time+60000*15:\n",
    "\n",
    "            time_interval=timestamp_to_datetime_US(start_time)+\"~\"+timestamp_to_datetime_US(int(t[-1]))\n",
    "\n",
    "            end = time.perf_counter()\n",
    "            time_with_loss[time_interval]={'loss':loss,\n",
    "                                \n",
    "                                          'nodes_count':len(unique_nodes),\n",
    "                                          'total_edges':total_edges,\n",
    "                                          'costed_time':(end-start)}\n",
    "            \n",
    "            \n",
    "            log=open(path+\"/\"+time_interval+\".txt\",'w')\n",
    "\n",
    "            \n",
    "            for e in edge_list: \n",
    "                loss+=e['loss']\n",
    "\n",
    "            loss=loss/event_count   \n",
    "            print(f'Time: {time_interval}, Loss: {loss:.4f}, Nodes_count: {len(unique_nodes)}, Cost Time: {(end-start):.2f}s')\n",
    "            edge_list = sorted(edge_list, key=lambda x:x['loss'],reverse=True)  \n",
    "            for e in edge_list: \n",
    "                log.write(str(e))\n",
    "                log.write(\"\\n\") \n",
    "            event_count=0\n",
    "            total_loss=0\n",
    "            loss=0\n",
    "            start_time=t[-1]\n",
    "            log.close()\n",
    "            edge_list.clear()\n",
    "            \n",
    " \n",
    "    return time_with_loss\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 9-22 hosts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load the models\n",
    "model=torch.load(\"./models/model_saved_traindata=hosts_9_22.pt\")\n",
    "memory,gnn, link_pred,neighbor_loader=model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ans_9_22_h201=test_day_new(graph_9_22_h201,\"graph_9_22_h201\",node_uuid2index_9_22_h201)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ans_9_22_h402=test_day_new(graph_9_22_h402,\"graph_9_22_h402\",node_uuid2index_9_22_h402)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ans_9_22_h660=test_day_new(graph_9_22_h660,\"graph_9_22_h660\",node_uuid2index_9_22_h660)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ans_9_22_h501=test_day_new(graph_9_22_h501,\"graph_9_22_h501\",node_uuid2index_9_22_h501)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ans_9_22_h051=test_day_new(graph_9_22_h051,\"graph_9_22_h051\",node_uuid2index_9_22_h051)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ans_9_22_h209=test_day_new(graph_9_22_h209,\"graph_9_22_h209\",node_uuid2index_9_22_h209)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 9-23~25 hosts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load the models\n",
    "model=torch.load(\"./models/model_saved_traindata=hosts_9_22.pt\")\n",
    "memory,gnn, link_pred,neighbor_loader=model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "graph_9_23_h201=test_day_new(graph_9_23_h201,\"graph_9_23_h201\",node_uuid2index_9_23_h201)\n",
    "graph_9_23_h402=test_day_new(graph_9_23_h402,\"graph_9_23_h402\",node_uuid2index_9_23_h402)\n",
    "graph_9_23_h660=test_day_new(graph_9_23_h660,\"graph_9_23_h660\",node_uuid2index_9_23_h660)\n",
    "graph_9_23_h501=test_day_new(graph_9_23_h501,\"graph_9_23_h501\",node_uuid2index_9_23_h501)\n",
    "graph_9_23_h051=test_day_new(graph_9_23_h051,\"graph_9_23_h051\",node_uuid2index_9_23_h051)\n",
    "graph_9_23_h207=test_day_new(graph_9_23_h207,\"graph_9_23_h207\",node_uuid2index_9_23_h207)\n",
    "\n",
    "graph_9_24_h201=test_day_new(graph_9_24_h201,\"graph_9_24_h201\",node_uuid2index_9_24_h201)\n",
    "graph_9_24_h402=test_day_new(graph_9_24_h402,\"graph_9_24_h402\",node_uuid2index_9_24_h402)\n",
    "graph_9_24_h660=test_day_new(graph_9_24_h660,\"graph_9_24_h660\",node_uuid2index_9_24_h660)\n",
    "graph_9_24_h501=test_day_new(graph_9_24_h501,\"graph_9_24_h501\",node_uuid2index_9_24_h501)\n",
    "graph_9_24_h051=test_day_new(graph_9_24_h051,\"graph_9_24_h051\",node_uuid2index_9_24_h051)\n",
    "graph_9_24_h207=test_day_new(graph_9_24_h207,\"graph_9_24_h207\",node_uuid2index_9_24_h207)\n",
    "\n",
    "graph_9_25_h201=test_day_new(graph_9_25_h201,\"graph_9_25_h201\",node_uuid2index_9_25_h201)\n",
    "graph_9_25_h402=test_day_new(graph_9_25_h402,\"graph_9_25_h402\",node_uuid2index_9_25_h402)\n",
    "graph_9_25_h660=test_day_new(graph_9_25_h660,\"graph_9_25_h660\",node_uuid2index_9_25_h660)\n",
    "graph_9_25_h501=test_day_new(graph_9_25_h501,\"graph_9_25_h501\",node_uuid2index_9_25_h501)\n",
    "graph_9_25_h051=test_day_new(graph_9_25_h051,\"graph_9_25_h051\",node_uuid2index_9_25_h051)\n",
    "graph_9_25_h207=test_day_new(graph_9_25_h207,\"graph_9_25_h207\",node_uuid2index_9_25_h207)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(graph_9_23_h201,\"./test_res/graph_9_23_h201\")\n",
    "torch.save(graph_9_23_h402,\"./test_res/graph_9_23_h402\")\n",
    "torch.save(graph_9_23_h660,\"./test_res/graph_9_23_h660\")\n",
    "torch.save(graph_9_23_h501,\"./test_res/graph_9_23_h501\")\n",
    "torch.save(graph_9_23_h051,\"./test_res/graph_9_23_h051\")\n",
    "torch.save(graph_9_23_h207,\"./test_res/graph_9_23_h207\")\n",
    "\n",
    "torch.save(graph_9_24_h201,\"./test_res/graph_9_24_h201\")\n",
    "torch.save(graph_9_24_h402,\"./test_res/graph_9_24_h402\")\n",
    "torch.save(graph_9_24_h660,\"./test_res/graph_9_24_h660\")\n",
    "torch.save(graph_9_24_h501,\"./test_res/graph_9_24_h501\")\n",
    "torch.save(graph_9_24_h051,\"./test_res/graph_9_24_h051\")\n",
    "torch.save(graph_9_24_h207,\"./test_res/graph_9_24_h207\")\n",
    "\n",
    "torch.save(graph_9_25_h201,\"./test_res/graph_9_25_h201\")\n",
    "torch.save(graph_9_25_h402,\"./test_res/graph_9_25_h402\")\n",
    "torch.save(graph_9_25_h660,\"./test_res/graph_9_25_h660\")\n",
    "torch.save(graph_9_25_h501,\"./test_res/graph_9_25_h501\")\n",
    "torch.save(graph_9_25_h051,\"./test_res/graph_9_25_h051\")\n",
    "torch.save(graph_9_25_h207,\"./test_res/graph_9_25_h207\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Compute the anomalous score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cal_train_IDF(find_str,file_list):\n",
    "    include_count=0\n",
    "    for f_path in (file_list):\n",
    "        f=open(f_path)\n",
    "        if find_str in f.read():\n",
    "            include_count+=1             \n",
    "    IDF=math.log(len(file_list)/(include_count+1))\n",
    "    return IDF\n",
    "\n",
    "\n",
    "def cal_IDF(find_str,file_path,file_list):\n",
    "    file_list=os.listdir(file_path)\n",
    "    include_count=0\n",
    "    different_neighbor=set()\n",
    "    for f_path in (file_list):\n",
    "        f=open(file_path+f_path)\n",
    "        if find_str in f.read():\n",
    "            include_count+=1\n",
    "                \n",
    "    IDF=math.log(len(file_list)/(include_count+1))\n",
    "    \n",
    "    return IDF,1\n",
    "\n",
    "\n",
    "def cal_IDF_by_file_in_mem(find_str,file_list):\n",
    "    include_count=0\n",
    "    different_neighbor=set()\n",
    "    for f in (file_list):       \n",
    "        if find_str in f:\n",
    "            include_count+=1\n",
    "    IDF=math.log(len(file_list)/(include_count+1))    \n",
    "    return IDF\n",
    "\n",
    "def cal_redundant(find_str,edge_list):\n",
    "    \n",
    "    different_neighbor=set()\n",
    "    for e in edge_list:\n",
    "        if find_str in str(e):\n",
    "            different_neighbor.add(e[0])\n",
    "            different_neighbor.add(e[1])\n",
    "    return len(different_neighbor)-2\n",
    "\n",
    "def cal_anomaly_loss(loss_list,edge_list,file_path):\n",
    "    \n",
    "    if len(loss_list)!=len(edge_list):\n",
    "        print(\"error!\")\n",
    "        return 0\n",
    "    count=0\n",
    "    loss_sum=0\n",
    "    loss_std=std(loss_list)\n",
    "    loss_mean=mean(loss_list)\n",
    "    edge_set=set()\n",
    "    node_set=set()\n",
    "    node2redundant={}\n",
    "    \n",
    "    thr=loss_mean+2.5*loss_std\n",
    "\n",
    "    print(\"thr:\",thr)\n",
    "    \n",
    "    for i in range(len(loss_list)):\n",
    "        if loss_list[i]>thr:\n",
    "            count+=1\n",
    "            src_node=edge_list[i][0]\n",
    "            dst_node=edge_list[i][1]\n",
    "\n",
    "            loss_sum+=loss_list[i]\n",
    "    \n",
    "            node_set.add(src_node)\n",
    "            node_set.add(dst_node)\n",
    "            edge_set.add(edge_list[i][0]+edge_list[i][1])\n",
    "    return count, loss_sum/count,node_set,edge_set\n",
    "#     return count, count/len(loss_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Construct the relations between time windows"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "    \n",
    "file_list=[]\n",
    "\n",
    "file_path=\"graph_9_22_h201/\"\n",
    "file_l=os.listdir(\"graph_9_22_h201/\")\n",
    "for i in file_l:\n",
    "    file_list.append(file_path+i)\n",
    "\n",
    "file_path=\"graph_9_22_h402/\"\n",
    "file_l=os.listdir(\"graph_9_22_h402/\")\n",
    "for i in file_l:\n",
    "    file_list.append(file_path+i)\n",
    "    \n",
    "file_path=\"graph_9_22_h660/\"\n",
    "file_l=os.listdir(\"graph_9_22_h660/\")\n",
    "for i in file_l:\n",
    "    file_list.append(file_path+i)\n",
    "    \n",
    "file_path=\"graph_9_22_h501/\"\n",
    "file_l=os.listdir(\"graph_9_22_h501/\")\n",
    "for i in file_l:\n",
    "    file_list.append(file_path+i)\n",
    "    \n",
    "file_path=\"graph_9_22_h051/\"\n",
    "file_l=os.listdir(\"graph_9_22_h051/\")\n",
    "for i in file_l:\n",
    "    file_list.append(file_path+i)    \n",
    "    \n",
    "    \n",
    "file_path=\"graph_9_22_h209/\"\n",
    "file_l=os.listdir(\"graph_9_22_h209/\")\n",
    "for i in file_l:\n",
    "    file_list.append(file_path+i) \n",
    "\n",
    "    \n",
    "def is_include_key_word(s):\n",
    "    keywords=[\n",
    "         '->',\n",
    "        '.DLL',\n",
    "        '.dll',\n",
    "        '.dat', \n",
    "       '.DAT', \n",
    "        'CACHE',\n",
    "        'Cache',\n",
    "        '.docx',\n",
    "        '.lnk',\n",
    "        '.LNK',\n",
    "        '.pptx',\n",
    "        '.xlsx',\n",
    "        'CVR',\n",
    "        'cvr',\n",
    "        'ZLEAZER',\n",
    "        'zleazer',\n",
    "        'SOFTWAREPROTECTIONPLATFORM',\n",
    "        'documents',\n",
    "        '.log',\n",
    "        '.nls',\n",
    "        '.EVTX',\n",
    "        '.evtx',\n",
    "        '.tmp',\n",
    "        '.TMP',\n",
    "        'Windows/Logs/',\n",
    "        'Windows/system32/',\n",
    "        'Windows/System32/',\n",
    "        '/Temp/',\n",
    "        'Users',\n",
    "        'USERS',\n",
    "        'Program Files',\n",
    "        'WINDOWS',\n",
    "        'Windows',\n",
    "        '$SII',\n",
    "        'svchost.exe',\n",
    "        'gpscript.exe',\n",
    "        'python.exe',\n",
    "        'rundll32.exe',\n",
    "        'consent.exe',\n",
    "        'python27',\n",
    "        'Python27',\n",
    "      ]\n",
    "    flag=False\n",
    "    for i in keywords:\n",
    "        if i in s:\n",
    "            flag=True\n",
    "    return flag    \n",
    "    \n",
    "    \n",
    "def cal_set_rel(s1,s2):\n",
    "    new_s=s1 & s2\n",
    "    count=0\n",
    "    for i in new_s:\n",
    "        if is_include_key_word(i) is not True:\n",
    "\n",
    "            if i in node_IDF.keys():\n",
    "                IDF=node_IDF[i]\n",
    "            else:\n",
    "                IDF=math.log(len(file_list)/(1))\n",
    "            if IDF>(math.log(len(file_list)*0.9)):\n",
    "                print(\"node:\",i,\" IDF:\",IDF)\n",
    "                count+=1\n",
    "    return count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "math.log(len(file_list)/(1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Compute the IDF-1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "file_list=[]\n",
    "\n",
    "file_path=\"graph_9_22_h201/\"\n",
    "file_l=os.listdir(\"graph_9_22_h201/\")\n",
    "for i in file_l:\n",
    "    file_list.append(file_path+i)\n",
    "\n",
    "file_path=\"graph_9_22_h402/\"\n",
    "file_l=os.listdir(\"graph_9_22_h402/\")\n",
    "for i in file_l:\n",
    "    file_list.append(file_path+i)\n",
    "    \n",
    "file_path=\"graph_9_22_h660/\"\n",
    "file_l=os.listdir(\"graph_9_22_h660/\")\n",
    "for i in file_l:\n",
    "    file_list.append(file_path+i)\n",
    "    \n",
    "file_path=\"graph_9_22_h501/\"\n",
    "file_l=os.listdir(\"graph_9_22_h501/\")\n",
    "for i in file_l:\n",
    "    file_list.append(file_path+i)\n",
    "    \n",
    "file_path=\"graph_9_22_h051/\"\n",
    "file_l=os.listdir(\"graph_9_22_h051/\")\n",
    "for i in file_l:\n",
    "    file_list.append(file_path+i)    \n",
    "    \n",
    "    \n",
    "file_path=\"graph_9_22_h209/\"\n",
    "file_l=os.listdir(\"graph_9_22_h209/\")\n",
    "for i in file_l:\n",
    "    file_list.append(file_path+i)    \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "node_set=set()\n",
    "\n",
    "for f_path in tqdm(file_list):\n",
    "    f=open(f_path)\n",
    "    for line in f:\n",
    "        l=line.strip()\n",
    "        jdata=eval(l)\n",
    "        if jdata['loss']>0:\n",
    "            if '->' not in str(jdata['srcmsg']):\n",
    "                node_set.add(str(jdata['srcmsg']).split(\"_@\")[-1])\n",
    "            if '->' not in str(jdata['dstmsg']):\n",
    "                node_set.add(str(jdata['dstmsg']).split(\"_@\")[-1]) \n",
    "\n",
    "\n",
    "node_list=list(node_set)\n",
    "del node_set\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "files_mem_list=[]\n",
    "for f_path in (file_list):\n",
    "        f=open(f_path)\n",
    "        files_mem_list.append(f.read())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_node_idf(_node_list,_files_mem_list,share_node_IDF):\n",
    "    for n in tqdm(_node_list):  \n",
    "        find_str=n\n",
    "        IDF=cal_IDF_by_file_in_mem(n,_files_mem_list)\n",
    "        share_node_IDF[n]=IDF\n",
    "\n",
    "import multiprocessing as mp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "share_node_IDF = mp.Manager().dict()\n",
    "cores=28\n",
    "offset=math.ceil(len(node_list)/cores)\n",
    "node_list_split=[]\n",
    "for i in range(0,len(node_list),offset):\n",
    "    node_list_split+=[node_list[i:i+offset]]     \n",
    "process_list=[]\n",
    "for i in range(cores):\n",
    "    process_list.append(mp.Process(target=process_node_idf, args=(node_list_split[i],files_mem_list,share_node_IDF)))\n",
    "for i in process_list:\n",
    "    i.start()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "node_IDF=dict(share_node_IDF)\n",
    "torch.save(node_IDF,\"node_IDF_9_22_hosts\")\n",
    "print(\"IDF weight calculate complete!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# force to terminate all process\n",
    "for i in process_list:\n",
    "    i.terminate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The IDF of the date 9-22 h201\n",
    "node_IDF={}\n",
    "node_set=set()\n",
    "\n",
    "file_list=[]\n",
    "\n",
    "file_path=\"graph_9_22/\"\n",
    "file_l=os.listdir(\"graph_9_22/\")\n",
    "\n",
    "\n",
    "for i in file_l:\n",
    "    file_list.append(file_path+i)\n",
    "\n",
    "for f_path in tqdm(file_list):\n",
    "    f=open(f_path)\n",
    "    for line in f:\n",
    "        l=line.strip()\n",
    "        jdata=eval(l)\n",
    "        if jdata['loss']>0:\n",
    "            if '->' not in str(jdata['srcmsg']):\n",
    "                node_set.add(str(jdata['srcmsg']))\n",
    "            if '->' not in str(jdata['dstmsg']):\n",
    "                node_set.add(str(jdata['dstmsg'])) \n",
    "\n",
    "for n in tqdm(node_set):\n",
    "#     find_str=list(eval(n).values())[0]\n",
    "    IDF=cal_train_IDF(n,file_list)\n",
    "    node_IDF[n]=IDF\n",
    "\n",
    "\n",
    "torch.save(node_IDF,\"node_IDF_9_22_without_netflow\")\n",
    "print(\"IDF weight calculate complete!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "# labelling"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "labels={}\n",
    "path=\"graph_9_23_h201/\"\n",
    "filelist = os.listdir(path)\n",
    "filelist.sort()\n",
    "for f in filelist:\n",
    "    labels[path+f]=0\n"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## h201"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "label_h201={}\n",
    "\n",
    "test_path=\"graph_9_23_h201/\"\n",
    "\n",
    "filelist = os.listdir(test_path)\n",
    "filelist.sort()\n",
    "for f in filelist:\n",
    "    label_h201[test_path+f]=0\n",
    "\n",
    "test_path=\"graph_9_24_h201/\"\n",
    "\n",
    "filelist = os.listdir(test_path)\n",
    "filelist.sort()\n",
    "for f in filelist:\n",
    "    label_h201[test_path+f]=0\n",
    "\n",
    "test_path=\"graph_9_25_h201/\"\n",
    "\n",
    "filelist = os.listdir(test_path)\n",
    "filelist.sort()\n",
    "for f in filelist:\n",
    "    label_h201[test_path+f]=0\n",
    "\n",
    "attack_list=[\n",
    "'graph_9_23_h201/2019-09-23 11:23:44.136~2019-09-23 11:38:30.698.txt',\n",
    " 'graph_9_23_h201/2019-09-23 11:38:40.698~2019-09-23 11:53:39.57.txt',\n",
    "\n",
    " 'graph_9_23_h201/2019-09-23 12:38:24.95~2019-09-23 12:54:14.286.txt',\n",
    " 'graph_9_23_h201/2019-09-23 12:55:28.286~2019-09-23 13:09:50.95.txt',\n",
    " 'graph_9_23_h201/2019-09-23 13:10:24.95~2019-09-23 13:24:56.43.txt',\n",
    "\n",
    "]\n",
    "for i in attack_list:\n",
    "    label_h201[i]=1"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## h402"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "label_h402={}\n",
    "\n",
    "test_path=\"graph_9_23_h402/\"\n",
    "\n",
    "filelist = os.listdir(test_path)\n",
    "filelist.sort()\n",
    "for f in filelist:\n",
    "    label_h402[test_path+f]=0\n",
    "\n",
    "test_path=\"graph_9_24_h402/\"\n",
    "\n",
    "filelist = os.listdir(test_path)\n",
    "filelist.sort()\n",
    "for f in filelist:\n",
    "    label_h402[test_path+f]=0\n",
    "\n",
    "test_path=\"graph_9_25_h402/\"\n",
    "\n",
    "filelist = os.listdir(test_path)\n",
    "filelist.sort()\n",
    "for f in filelist:\n",
    "    label_h402[test_path+f]=0\n",
    "\n",
    "attack_list=[\n",
    " 'graph_9_23_h402/2019-09-23 13:10:24.429~2019-09-23 13:25:09.374.txt',\n",
    " 'graph_9_23_h402/2019-09-23 13:25:20.374~2019-09-23 13:40:21.268.txt',\n",
    "\n",
    "    # adjust\n",
    "#      'graph_9_23_h402/2019-09-23 13:40:16.268~2019-09-23 13:55:31.31.txt',\n",
    "#  'graph_9_23_h402/2019-09-23 13:55:12.31~2019-09-23 14:10:58.2.txt',\n",
    "]\n",
    "for i in attack_list:\n",
    "    label_h402[i]=1"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## h660"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "label_h660={}\n",
    "\n",
    "test_path=\"graph_9_23_h660/\"\n",
    "\n",
    "filelist = os.listdir(test_path)\n",
    "filelist.sort()\n",
    "for f in filelist:\n",
    "    label_h660[test_path+f]=0\n",
    "\n",
    "test_path=\"graph_9_24_h660/\"\n",
    "\n",
    "filelist = os.listdir(test_path)\n",
    "filelist.sort()\n",
    "for f in filelist:\n",
    "    label_h660[test_path+f]=0\n",
    "\n",
    "test_path=\"graph_9_25_h660/\"\n",
    "\n",
    "filelist = os.listdir(test_path)\n",
    "filelist.sort()\n",
    "for f in filelist:\n",
    "    label_h660[test_path+f]=0\n",
    "\n",
    "attack_list=[\n",
    "'graph_9_23_h660/2019-09-23 13:27:28.512~2019-09-23 13:42:30.682.txt',\n",
    " 'graph_9_23_h660/2019-09-23 13:42:24.682~2019-09-23 13:57:57.566.txt',\n",
    " 'graph_9_23_h660/2019-09-23 13:57:20.566~2019-09-23 14:12:59.139.txt',\n",
    "]\n",
    "for i in attack_list:\n",
    "    label_h660[i]=1"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "label_h660"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## h501"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "label_h501={}\n",
    "\n",
    "test_path=\"graph_9_23_h501/\"\n",
    "\n",
    "filelist = os.listdir(test_path)\n",
    "filelist.sort()\n",
    "for f in filelist:\n",
    "    label_h501[test_path+f]=0\n",
    "\n",
    "test_path=\"graph_9_24_h501/\"\n",
    "\n",
    "filelist = os.listdir(test_path)\n",
    "filelist.sort()\n",
    "for f in filelist:\n",
    "    label_h501[test_path+f]=0\n",
    "\n",
    "test_path=\"graph_9_25_h501/\"\n",
    "\n",
    "filelist = os.listdir(test_path)\n",
    "filelist.sort()\n",
    "for f in filelist:\n",
    "    label_h501[test_path+f]=0\n",
    "\n",
    "attack_list=[\n",
    "\n",
    " 'graph_9_24_h501/2019-09-24 10:15:28.241~2019-09-24 10:30:00.201.txt',\n",
    " 'graph_9_24_h501/2019-09-24 10:30:24.201~2019-09-24 10:45:02.7.txt',\n",
    " 'graph_9_24_h501/2019-09-24 10:45:20.7~2019-09-24 11:00:31.385.txt',\n",
    " 'graph_9_24_h501/2019-09-24 11:00:16.385~2019-09-24 11:16:09.755.txt',\n",
    " 'graph_9_24_h501/2019-09-24 11:15:12.755~2019-09-24 11:31:14.287.txt',\n",
    " 'graph_9_24_h501/2019-09-24 11:32:16.287~2019-09-24 11:46:31.541.txt',\n",
    "\n",
    " 'graph_9_24_h501/2019-09-24 13:04:00.804~2019-09-24 13:17:29.451.txt',\n",
    " 'graph_9_24_h501/2019-09-24 13:18:56.451~2019-09-24 13:32:46.454.txt',\n",
    " 'graph_9_24_h501/2019-09-24 13:33:52.454~2019-09-24 13:48:02.493.txt',\n",
    "\n",
    "# adjust\n",
    "\n",
    "#  'graph_9_24_h501/2019-09-24 11:47:12.541~2019-09-24 12:01:32.699.txt',\n",
    "#  'graph_9_24_h501/2019-09-24 12:02:08.699~2019-09-24 12:16:34.904.txt',\n",
    "#  'graph_9_24_h501/2019-09-24 12:32:00.112~2019-09-24 12:46:58.691.txt',\n",
    "#  'graph_9_24_h501/2019-09-24 12:46:56.691~2019-09-24 13:02:02.804.txt',\n",
    "#  'graph_9_24_h501/2019-09-24 16:22:24.281~2019-09-24 16:36:06.878.txt',\n",
    "\n",
    "]\n",
    "for i in attack_list:\n",
    "    label_h501[i]=1"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "label_h501"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## h051"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "label_h051={}\n",
    "\n",
    "test_path=\"graph_9_23_h051/\"\n",
    "\n",
    "filelist = os.listdir(test_path)\n",
    "filelist.sort()\n",
    "for f in filelist:\n",
    "    label_h051[test_path+f]=0\n",
    "\n",
    "test_path=\"graph_9_24_h051/\"\n",
    "\n",
    "filelist = os.listdir(test_path)\n",
    "filelist.sort()\n",
    "for f in filelist:\n",
    "    label_h051[test_path+f]=0\n",
    "\n",
    "test_path=\"graph_9_25_h051/\"\n",
    "\n",
    "filelist = os.listdir(test_path)\n",
    "filelist.sort()\n",
    "for f in filelist:\n",
    "    label_h051[test_path+f]=0\n",
    "\n",
    "attack_list=[\n",
    "'graph_9_25_h051/2019-09-25 10:26:08.397~2019-09-25 10:41:40.247.txt',\n",
    " 'graph_9_25_h051/2019-09-25 10:41:04.247~2019-09-25 10:56:56.92.txt',\n",
    " 'graph_9_25_h051/2019-09-25 10:56:00.92~2019-09-25 11:12:03.608.txt',\n",
    "]\n",
    "for i in attack_list:\n",
    "    label_h051[i]=1"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "label_h051"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## h207"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "label_h207={}\n",
    "\n",
    "test_path=\"graph_9_23_h207/\"\n",
    "\n",
    "filelist = os.listdir(test_path)\n",
    "filelist.sort()\n",
    "for f in filelist:\n",
    "    label_h207[test_path+f]=0\n",
    "\n",
    "test_path=\"graph_9_24_h207/\"\n",
    "\n",
    "filelist = os.listdir(test_path)\n",
    "filelist.sort()\n",
    "for f in filelist:\n",
    "    label_h207[test_path+f]=0\n",
    "\n",
    "test_path=\"graph_9_25_h207/\"\n",
    "\n",
    "filelist = os.listdir(test_path)\n",
    "filelist.sort()\n",
    "for f in filelist:\n",
    "    label_h207[test_path+f]=0\n",
    "\n",
    "attack_list=[\n",
    "\n",
    "]\n",
    "for i in attack_list:\n",
    "    label_h207[i]=1"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Anoamly detection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "def classifier_evaluation(y_test, y_test_pred):\n",
    "    # groundtruth, pred_value\n",
    "    tn, fp, fn, tp =confusion_matrix(y_test, y_test_pred).ravel()\n",
    "#     tn+=100\n",
    "#     print(clf_name,\" : \")\n",
    "    print('tn:',tn)\n",
    "    print('fp:',fp)\n",
    "    print('fn:',fn)\n",
    "    print('tp:',tp)\n",
    "    precision=tp/(tp+fp)\n",
    "    recall=tp/(tp+fn)\n",
    "    accuracy=(tp+tn)/(tp+tn+fp+fn)\n",
    "    fscore=2*(precision*recall)/(precision+recall)\n",
    "    auc_val=roc_auc_score(y_test, y_test_pred)\n",
    "    print(\"precision:\",precision)\n",
    "    print(\"recall:\",recall)\n",
    "    print(\"fscore:\",fscore)\n",
    "    print(\"accuracy:\",accuracy)\n",
    "    print(\"auc_val:\",auc_val)\n",
    "    return precision,recall,fscore,accuracy,auc_val"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## h201"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 9-23"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "\n",
    "node_IDF=torch.load(\"node_IDF_9_22_hosts\")\n",
    "\n",
    "# node_set_list=[]\n",
    "history_list_9_23_h201=[]\n",
    "tw_que=[]\n",
    "his_tw={}\n",
    "current_tw={}\n",
    "\n",
    "test_path=\"graph_9_23_h201/\"\n",
    "\n",
    "\n",
    "file_l=os.listdir(test_path)\n",
    "file_l.sort()\n",
    "index_count=0\n",
    "for f_path in (file_l):\n",
    "    f=open(test_path+f_path)\n",
    "    edge_loss_list=[]\n",
    "    edge_list=[]\n",
    "    print('index_count:',index_count)\n",
    "    \n",
    "    for line in f:\n",
    "        l=line.strip()\n",
    "        jdata=eval(l)\n",
    "        edge_loss_list.append(jdata['loss'])\n",
    "        edge_list.append([str(jdata['srcmsg']).split(\"_@\")[-1],str(jdata['dstmsg']).split(\"_@\")[-1]])\n",
    "#         edge_list.append([str(jdata['srcmsg']),str(jdata['dstmsg'])])\n",
    "        \n",
    "#     df_list_9_22.append(pd.DataFrame(edge_loss_list))\n",
    "    count,loss_avg,node_set,edge_set=cal_anomaly_loss(edge_loss_list,edge_list,test_path)\n",
    "    current_tw['name']=f_path\n",
    "    current_tw['loss']=loss_avg\n",
    "    current_tw['index']=index_count\n",
    "    current_tw['nodeset']=node_set\n",
    "\n",
    "    added_que_flag=False\n",
    "    for hq in history_list_9_23_h201:\n",
    "        for his_tw in hq:\n",
    "            if cal_set_rel(current_tw['nodeset'],his_tw['nodeset'])!=0 and current_tw['name']!=his_tw['name']:\n",
    "                hq.append(copy.deepcopy(current_tw))\n",
    "                print(f\"{his_tw['name']=}\")\n",
    "                added_que_flag=True\n",
    "                break\n",
    "            if added_que_flag:\n",
    "                break\n",
    "    if added_que_flag is False:\n",
    "        temp_hq=[copy.deepcopy(current_tw)]\n",
    "        history_list_9_23_h201.append(temp_hq)\n",
    "  \n",
    "    index_count+=1\n",
    "#     node_set_list.append(node_set)\n",
    "    print( f_path,\"  \",loss_avg,\" count:\",count,\" percentage:\",count/len(edge_list),\" node count:\",len(node_set),\" edge count:\",len(edge_set))\n",
    "#     y_data_4_10.append([loss_avg,labels_4_10[f_path],f_path])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_label_h201={}\n",
    "\n",
    "test_path=\"graph_9_23_h201/\"\n",
    "    \n",
    "filelist = os.listdir(test_path)\n",
    "filelist.sort()\n",
    "for f in filelist:\n",
    "    pred_label_h201[test_path+f]=0\n",
    "    \n",
    "test_path=\"graph_9_24_h201/\"\n",
    "    \n",
    "filelist = os.listdir(test_path)\n",
    "filelist.sort()\n",
    "for f in filelist:\n",
    "    pred_label_h201[test_path+f]=0\n",
    "    \n",
    "test_path=\"graph_9_25_h201/\"\n",
    "    \n",
    "filelist = os.listdir(test_path)\n",
    "filelist.sort()\n",
    "for f in filelist:\n",
    "    pred_label_h201[test_path+f]=0\n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "for hl in history_list_9_23_h201:\n",
    "    loss_count=0\n",
    "    for hq in hl:\n",
    "        if loss_count==0:\n",
    "            loss_count=(loss_count+1)*(hq['loss']+1)\n",
    "        else:\n",
    "            loss_count=(loss_count)*(hq['loss']+1)\n",
    "    name_list=[]\n",
    "    if loss_count>1000:\n",
    "        name_list=[]\n",
    "        for i in hl:\n",
    "            name_list.append(i['name'])\n",
    "        print(*name_list, sep = \"\\n\")\n",
    "        for i in name_list:\n",
    "            pred_label_h201[\"graph_9_23_h201/\"+i]=1\n",
    "        print(loss_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# evaluate\n",
    "y=[]\n",
    "y_pred=[]\n",
    "for i in label_h201:\n",
    "    y.append(label_h201[i])\n",
    "    y_pred.append(pred_label_h201[i])\n",
    "    \n",
    "classifier_evaluation(y,y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 9-24"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "node_IDF=torch.load(\"node_IDF_9_22_hosts\")\n",
    "\n",
    "# node_set_list=[]\n",
    "history_list=[]\n",
    "tw_que=[]\n",
    "his_tw={}\n",
    "current_tw={}\n",
    "\n",
    "test_path=\"graph_9_24_h201/\"\n",
    "\n",
    "\n",
    "file_l=os.listdir(test_path)\n",
    "file_l.sort()\n",
    "index_count=0\n",
    "for f_path in (file_l):\n",
    "    f=open(test_path+f_path)\n",
    "    edge_loss_list=[]\n",
    "    edge_list=[]\n",
    "    print('index_count:',index_count)\n",
    "    \n",
    "    for line in f:\n",
    "        l=line.strip()\n",
    "        jdata=eval(l)\n",
    "        edge_loss_list.append(jdata['loss'])\n",
    "        edge_list.append([str(jdata['srcmsg']).split(\"_@\")[-1],str(jdata['dstmsg']).split(\"_@\")[-1]])\n",
    "#         edge_list.append([str(jdata['srcmsg']),str(jdata['dstmsg'])])\n",
    "        \n",
    "#     df_list_9_22.append(pd.DataFrame(edge_loss_list))\n",
    "    count,loss_avg,node_set,edge_set=cal_anomaly_loss(edge_loss_list,edge_list,test_path)\n",
    "    current_tw['name']=f_path\n",
    "    current_tw['loss']=loss_avg\n",
    "    current_tw['index']=index_count\n",
    "    current_tw['nodeset']=node_set\n",
    "\n",
    "    added_que_flag=False\n",
    "    for hq in history_list:\n",
    "        for his_tw in hq:\n",
    "            if cal_set_rel(current_tw['nodeset'],his_tw['nodeset'])!=0 and current_tw['name']!=his_tw['name']:\n",
    "                hq.append(copy.deepcopy(current_tw))\n",
    "                print(f\"{his_tw['name']=}\")\n",
    "                added_que_flag=True\n",
    "                break\n",
    "            if added_que_flag:\n",
    "                break\n",
    "    if added_que_flag is False:\n",
    "        temp_hq=[copy.deepcopy(current_tw)]\n",
    "        history_list.append(temp_hq)\n",
    "  \n",
    "    index_count+=1\n",
    "#     node_set_list.append(node_set)\n",
    "    print( f_path,\"  \",loss_avg,\" count:\",count,\" percentage:\",count/len(edge_list),\" node count:\",len(node_set),\" edge count:\",len(edge_set))\n",
    "#     y_data_4_10.append([loss_avg,labels_4_10[f_path],f_path])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "for hl in history_list:\n",
    "    loss_count=0\n",
    "    for hq in hl:\n",
    "        if loss_count==0:\n",
    "            loss_count=(loss_count+1)*(hq['loss']+1)\n",
    "        else:\n",
    "            loss_count=(loss_count)*(hq['loss']+1)\n",
    "    name_list=[]\n",
    "    if loss_count>10000:\n",
    "        name_list=[]\n",
    "        for i in hl:\n",
    "            name_list.append(i['name'])\n",
    "        print(name_list)\n",
    "#         for i in name_list:\n",
    "#             pred_label[i]=1\n",
    "        print(loss_count)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 9-25"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "node_IDF=torch.load(\"node_IDF_9_22_hosts\")\n",
    "\n",
    "# node_set_list=[]\n",
    "history_list=[]\n",
    "tw_que=[]\n",
    "his_tw={}\n",
    "current_tw={}\n",
    "\n",
    "test_path=\"graph_9_25_h201/\"\n",
    "\n",
    "\n",
    "file_l=os.listdir(test_path)\n",
    "file_l.sort()\n",
    "index_count=0\n",
    "for f_path in (file_l):\n",
    "    f=open(test_path+f_path)\n",
    "    edge_loss_list=[]\n",
    "    edge_list=[]\n",
    "    print('index_count:',index_count)\n",
    "    \n",
    "    for line in f:\n",
    "        l=line.strip()\n",
    "        jdata=eval(l)\n",
    "        edge_loss_list.append(jdata['loss'])\n",
    "        edge_list.append([str(jdata['srcmsg']).split(\"_@\")[-1],str(jdata['dstmsg']).split(\"_@\")[-1]])\n",
    "#         edge_list.append([str(jdata['srcmsg']),str(jdata['dstmsg'])])\n",
    "    count,loss_avg,node_set,edge_set=cal_anomaly_loss(edge_loss_list,edge_list,test_path)\n",
    "    current_tw['name']=f_path\n",
    "    current_tw['loss']=loss_avg\n",
    "    current_tw['index']=index_count\n",
    "    current_tw['nodeset']=node_set\n",
    "\n",
    "    added_que_flag=False\n",
    "    for hq in history_list:\n",
    "        for his_tw in hq:\n",
    "            if cal_set_rel(current_tw['nodeset'],his_tw['nodeset'])!=0 and current_tw['name']!=his_tw['name']:\n",
    "                hq.append(copy.deepcopy(current_tw))\n",
    "                print(f\"{his_tw['name']=}\")\n",
    "                added_que_flag=True\n",
    "                break\n",
    "            if added_que_flag:\n",
    "                break\n",
    "    if added_que_flag is False:\n",
    "        temp_hq=[copy.deepcopy(current_tw)]\n",
    "        history_list.append(temp_hq)\n",
    "  \n",
    "    index_count+=1\n",
    "\n",
    "    print( f_path,\"  \",loss_avg,\" count:\",count,\" percentage:\",count/len(edge_list),\" node count:\",len(node_set),\" edge count:\",len(edge_set))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# pred_label={}\n",
    "\n",
    "    \n",
    "# filelist = os.listdir(\"graph_9_23\")\n",
    "# for f in filelist:\n",
    "#     pred_label[f]=0\n",
    "\n",
    "\n",
    "for hl in history_list:\n",
    "    loss_count=0\n",
    "    for hq in hl:\n",
    "        if loss_count==0:\n",
    "            loss_count=(loss_count+1)*(hq['loss']+1)\n",
    "        else:\n",
    "            loss_count=(loss_count)*(hq['loss']+1)\n",
    "    name_list=[]\n",
    "    if loss_count>10000:\n",
    "        name_list=[]\n",
    "        for i in hl:\n",
    "            name_list.append(i['name'])\n",
    "        print(name_list)\n",
    "#         for i in name_list:\n",
    "#             pred_label[i]=1\n",
    "        print(loss_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## h402"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "node_IDF=torch.load(\"node_IDF_9_22_hosts\")\n",
    "\n",
    "# node_set_list=[]\n",
    "history_list_9_23_h402=[]\n",
    "tw_que=[]\n",
    "his_tw={}\n",
    "current_tw={}\n",
    "\n",
    "test_path=\"graph_9_23_h402/\"\n",
    "\n",
    "\n",
    "file_l=os.listdir(test_path)\n",
    "file_l.sort()\n",
    "index_count=0\n",
    "for f_path in (file_l):\n",
    "    f=open(test_path+f_path)\n",
    "    edge_loss_list=[]\n",
    "    edge_list=[]\n",
    "    print('index_count:',index_count)\n",
    "    \n",
    "    for line in f:\n",
    "        l=line.strip()\n",
    "        jdata=eval(l)\n",
    "        edge_loss_list.append(jdata['loss'])\n",
    "        edge_list.append([str(jdata['srcmsg']).split(\"_@\")[-1],str(jdata['dstmsg']).split(\"_@\")[-1]])\n",
    "#         edge_list.append([str(jdata['srcmsg']),str(jdata['dstmsg'])])\n",
    "        \n",
    "#     df_list_9_22.append(pd.DataFrame(edge_loss_list))\n",
    "    count,loss_avg,node_set,edge_set=cal_anomaly_loss(edge_loss_list,edge_list,test_path)\n",
    "    current_tw['name']=f_path\n",
    "    current_tw['loss']=loss_avg\n",
    "    current_tw['index']=index_count\n",
    "    current_tw['nodeset']=node_set\n",
    "\n",
    "    added_que_flag=False\n",
    "    for hq in history_list_9_23_h402:\n",
    "        for his_tw in hq:\n",
    "            if cal_set_rel(current_tw['nodeset'],his_tw['nodeset'])!=0 and current_tw['name']!=his_tw['name']:\n",
    "                hq.append(copy.deepcopy(current_tw))\n",
    "                print(f\"{his_tw['name']=}\")\n",
    "                added_que_flag=True\n",
    "                break\n",
    "            if added_que_flag:\n",
    "                break\n",
    "    if added_que_flag is False:\n",
    "        temp_hq=[copy.deepcopy(current_tw)]\n",
    "        history_list_9_23_h402.append(temp_hq)\n",
    "  \n",
    "    index_count+=1\n",
    "#     node_set_list.append(node_set)\n",
    "    print( f_path,\"  \",loss_avg,\" count:\",count,\" percentage:\",count/len(edge_list),\" node count:\",len(node_set),\" edge count:\",len(edge_set))\n",
    "#     y_data_4_10.append([loss_avg,labels_4_10[f_path],f_path])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_label_h402={}\n",
    "\n",
    "test_path=\"graph_9_23_h402/\"\n",
    "    \n",
    "filelist = os.listdir(test_path)\n",
    "filelist.sort()\n",
    "for f in filelist:\n",
    "    pred_label_h402[test_path+f]=0\n",
    "    \n",
    "test_path=\"graph_9_24_h402/\"\n",
    "    \n",
    "filelist = os.listdir(test_path)\n",
    "filelist.sort()\n",
    "for f in filelist:\n",
    "    pred_label_h402[test_path+f]=0\n",
    "    \n",
    "test_path=\"graph_9_25_h402/\"\n",
    "    \n",
    "filelist = os.listdir(test_path)\n",
    "filelist.sort()\n",
    "for f in filelist:\n",
    "    pred_label_h402[test_path+f]=0\n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "for hl in history_list_9_23_h402:\n",
    "    loss_count=0\n",
    "    for hq in hl:\n",
    "        if loss_count==0:\n",
    "            loss_count=(loss_count+1)*(hq['loss']+1)\n",
    "        else:\n",
    "            loss_count=(loss_count)*(hq['loss']+1)\n",
    "    name_list=[]\n",
    "    if loss_count>1000:\n",
    "        name_list=[]\n",
    "        for i in hl:\n",
    "            name_list.append(i['name'])\n",
    "        print(*name_list, sep = \"\\n\")\n",
    "        for i in name_list:\n",
    "            pred_label_h402[\"graph_9_23_h402/\"+i]=1\n",
    "        print(loss_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Evalaute\n",
    "y=[]\n",
    "y_pred=[]\n",
    "for i in label_h402:\n",
    "    y.append(label_h402[i])\n",
    "    y_pred.append(pred_label_h402[i])\n",
    "    \n",
    "classifier_evaluation(y,y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## h660"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "node_IDF=torch.load(\"node_IDF_9_22_hosts\")\n",
    "\n",
    "# node_set_list=[]\n",
    "history_list_9_23_h660=[]\n",
    "tw_que=[]\n",
    "his_tw={}\n",
    "current_tw={}\n",
    "\n",
    "test_path=\"graph_9_23_h660/\"\n",
    "\n",
    "\n",
    "file_l=os.listdir(test_path)\n",
    "file_l.sort()\n",
    "index_count=0\n",
    "for f_path in (file_l):\n",
    "    f=open(test_path+f_path)\n",
    "    edge_loss_list=[]\n",
    "    edge_list=[]\n",
    "    print('index_count:',index_count)\n",
    "    \n",
    "    for line in f:\n",
    "        l=line.strip()\n",
    "        jdata=eval(l)\n",
    "        edge_loss_list.append(jdata['loss'])\n",
    "        edge_list.append([str(jdata['srcmsg']).split(\"_@\")[-1],str(jdata['dstmsg']).split(\"_@\")[-1]])\n",
    "#         edge_list.append([str(jdata['srcmsg']),str(jdata['dstmsg'])])\n",
    "        \n",
    "#     df_list_9_22.append(pd.DataFrame(edge_loss_list))\n",
    "    count,loss_avg,node_set,edge_set=cal_anomaly_loss(edge_loss_list,edge_list,test_path)\n",
    "    current_tw['name']=f_path\n",
    "    current_tw['loss']=loss_avg\n",
    "    current_tw['index']=index_count\n",
    "    current_tw['nodeset']=node_set\n",
    "\n",
    "    added_que_flag=False\n",
    "    for hq in history_list_9_23_h660:\n",
    "        for his_tw in hq:\n",
    "            if cal_set_rel(current_tw['nodeset'],his_tw['nodeset'])!=0 and current_tw['name']!=his_tw['name']:\n",
    "                hq.append(copy.deepcopy(current_tw))\n",
    "                print(f\"{his_tw['name']=}\")\n",
    "                added_que_flag=True\n",
    "                break\n",
    "            if added_que_flag:\n",
    "                break\n",
    "    if added_que_flag is False:\n",
    "        temp_hq=[copy.deepcopy(current_tw)]\n",
    "        history_list_9_23_h660.append(temp_hq)\n",
    "  \n",
    "    index_count+=1\n",
    "#     node_set_list.append(node_set)\n",
    "    print( f_path,\"  \",loss_avg,\" count:\",count,\" percentage:\",count/len(edge_list),\" node count:\",len(node_set),\" edge count:\",len(edge_set))\n",
    "#     y_data_4_10.append([loss_avg,labels_4_10[f_path],f_path])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_label_h660={}\n",
    "\n",
    "test_path=\"graph_9_23_h660/\"\n",
    "    \n",
    "filelist = os.listdir(test_path)\n",
    "filelist.sort()\n",
    "for f in filelist:\n",
    "    pred_label_h660[test_path+f]=0\n",
    "    \n",
    "test_path=\"graph_9_24_h660/\"\n",
    "    \n",
    "filelist = os.listdir(test_path)\n",
    "filelist.sort()\n",
    "for f in filelist:\n",
    "    pred_label_h660[test_path+f]=0\n",
    "    \n",
    "test_path=\"graph_9_25_h660/\"\n",
    "    \n",
    "filelist = os.listdir(test_path)\n",
    "filelist.sort()\n",
    "for f in filelist:\n",
    "    pred_label_h660[test_path+f]=0\n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "for hl in history_list_9_23_h660:\n",
    "    loss_count=0\n",
    "    for hq in hl:\n",
    "        if loss_count==0:\n",
    "            loss_count=(loss_count+1)*(hq['loss']+1)\n",
    "        else:\n",
    "            loss_count=(loss_count)*(hq['loss']+1)\n",
    "    name_list=[]\n",
    "    if loss_count>100:\n",
    "        name_list=[]\n",
    "        for i in hl:\n",
    "            name_list.append(i['name'])\n",
    "        print(*name_list, sep = \"\\n\")\n",
    "        for i in name_list:\n",
    "            pred_label_h660[\"graph_9_23_h660/\"+i]=1\n",
    "        print(loss_count)\n",
    "        \n",
    "        \n",
    "# evalute\n",
    "y=[]\n",
    "y_pred=[]\n",
    "for i in label_h660:\n",
    "    y.append(label_h660[i])\n",
    "    y_pred.append(pred_label_h660[i])\n",
    "    \n",
    "classifier_evaluation(y,y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## h501"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "node_IDF=torch.load(\"node_IDF_9_22_hosts\")\n",
    "\n",
    "# node_set_list=[]\n",
    "history_list_9_24_h501=[]\n",
    "tw_que=[]\n",
    "his_tw={}\n",
    "current_tw={}\n",
    "\n",
    "test_path=\"graph_9_24_h501/\"\n",
    "\n",
    "\n",
    "file_l=os.listdir(test_path)\n",
    "file_l.sort()\n",
    "index_count=0\n",
    "for f_path in (file_l):\n",
    "    f=open(test_path+f_path)\n",
    "    edge_loss_list=[]\n",
    "    edge_list=[]\n",
    "    print('index_count:',index_count)\n",
    "    \n",
    "    for line in f:\n",
    "        l=line.strip()\n",
    "        jdata=eval(l)\n",
    "        edge_loss_list.append(jdata['loss'])\n",
    "        edge_list.append([str(jdata['srcmsg']).split(\"_@\")[-1],str(jdata['dstmsg']).split(\"_@\")[-1]])\n",
    "#         edge_list.append([str(jdata['srcmsg']),str(jdata['dstmsg'])])\n",
    "        \n",
    "#     df_list_9_22.append(pd.DataFrame(edge_loss_list))\n",
    "    count,loss_avg,node_set,edge_set=cal_anomaly_loss(edge_loss_list,edge_list,test_path)\n",
    "    current_tw['name']=f_path\n",
    "    current_tw['loss']=loss_avg\n",
    "    current_tw['index']=index_count\n",
    "    current_tw['nodeset']=node_set\n",
    "\n",
    "    added_que_flag=False\n",
    "    for hq in history_list_9_24_h501:\n",
    "        for his_tw in hq:\n",
    "            if cal_set_rel(current_tw['nodeset'],his_tw['nodeset'])!=0 and current_tw['name']!=his_tw['name']:\n",
    "                hq.append(copy.deepcopy(current_tw))\n",
    "                print(f\"{his_tw['name']=}\")\n",
    "                added_que_flag=True\n",
    "                break\n",
    "            if added_que_flag:\n",
    "                break\n",
    "    if added_que_flag is False:\n",
    "        temp_hq=[copy.deepcopy(current_tw)]\n",
    "        history_list_9_24_h501.append(temp_hq)\n",
    "  \n",
    "    index_count+=1\n",
    "#     node_set_list.append(node_set)\n",
    "    print( f_path,\"  \",loss_avg,\" count:\",count,\" percentage:\",count/len(edge_list),\" node count:\",len(node_set),\" edge count:\",len(edge_set))\n",
    "#     y_data_4_10.append([loss_avg,labels_4_10[f_path],f_path])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_label_h501={}\n",
    "\n",
    "test_path=\"graph_9_23_h501/\"\n",
    "    \n",
    "filelist = os.listdir(test_path)\n",
    "filelist.sort()\n",
    "for f in filelist:\n",
    "    pred_label_h501[test_path+f]=0\n",
    "    \n",
    "test_path=\"graph_9_24_h501/\"\n",
    "    \n",
    "filelist = os.listdir(test_path)\n",
    "filelist.sort()\n",
    "for f in filelist:\n",
    "    pred_label_h501[test_path+f]=0\n",
    "    \n",
    "test_path=\"graph_9_25_h501/\"\n",
    "    \n",
    "filelist = os.listdir(test_path)\n",
    "filelist.sort()\n",
    "for f in filelist:\n",
    "    pred_label_h501[test_path+f]=0\n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "for hl in history_list_9_24_h501:\n",
    "    loss_count=0\n",
    "    for hq in hl:\n",
    "        if loss_count==0:\n",
    "            loss_count=(loss_count+1)*(hq['loss']+1)\n",
    "        else:\n",
    "            loss_count=(loss_count)*(hq['loss']+1)\n",
    "    name_list=[]\n",
    "    if loss_count>200:\n",
    "        name_list=[]\n",
    "        for i in hl:\n",
    "            name_list.append(i['name'])\n",
    "        print(*name_list, sep = \"\\n\")\n",
    "        for i in name_list:\n",
    "            pred_label_h501[\"graph_9_24_h501/\"+i]=1\n",
    "        print(loss_count)\n",
    "        \n",
    "        \n",
    "# evaluate\n",
    "y=[]\n",
    "y_pred=[]\n",
    "for i in label_h501:\n",
    "    y.append(label_h501[i])\n",
    "    y_pred.append(pred_label_h501[i])\n",
    "    \n",
    "classifier_evaluation(y,y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## h051"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# node_IDF=torch.load(\"node_IDF_4_6\")\n",
    "node_IDF=torch.load(\"node_IDF_9_22_hosts\")\n",
    "\n",
    "# node_set_list=[]\n",
    "history_list_9_25_h051=[]\n",
    "tw_que=[]\n",
    "his_tw={}\n",
    "current_tw={}\n",
    "\n",
    "test_path=\"graph_9_25_h051/\"\n",
    "\n",
    "\n",
    "file_l=os.listdir(test_path)\n",
    "file_l.sort()\n",
    "index_count=0\n",
    "for f_path in (file_l):\n",
    "    f=open(test_path+f_path)\n",
    "    edge_loss_list=[]\n",
    "    edge_list=[]\n",
    "    print('index_count:',index_count)\n",
    "    \n",
    "    for line in f:\n",
    "        l=line.strip()\n",
    "        jdata=eval(l)\n",
    "        edge_loss_list.append(jdata['loss'])\n",
    "        edge_list.append([str(jdata['srcmsg']).split(\"_@\")[-1],str(jdata['dstmsg']).split(\"_@\")[-1]])\n",
    "#         edge_list.append([str(jdata['srcmsg']),str(jdata['dstmsg'])])\n",
    "        \n",
    "#     df_list_9_22.append(pd.DataFrame(edge_loss_list))\n",
    "    count,loss_avg,node_set,edge_set=cal_anomaly_loss(edge_loss_list,edge_list,test_path)\n",
    "    current_tw['name']=f_path\n",
    "    current_tw['loss']=loss_avg\n",
    "    current_tw['index']=index_count\n",
    "    current_tw['nodeset']=node_set\n",
    "\n",
    "    added_que_flag=False\n",
    "    for hq in history_list_9_25_h051:\n",
    "        for his_tw in hq:\n",
    "            if cal_set_rel(current_tw['nodeset'],his_tw['nodeset'])!=0 and current_tw['name']!=his_tw['name']:\n",
    "                hq.append(copy.deepcopy(current_tw))\n",
    "                print(f\"{his_tw['name']=}\")\n",
    "                added_que_flag=True\n",
    "                break\n",
    "            if added_que_flag:\n",
    "                break\n",
    "    if added_que_flag is False:\n",
    "        temp_hq=[copy.deepcopy(current_tw)]\n",
    "        history_list_9_25_h051.append(temp_hq)\n",
    "  \n",
    "    index_count+=1\n",
    "#     node_set_list.append(node_set)\n",
    "    print( f_path,\"  \",loss_avg,\" count:\",count,\" percentage:\",count/len(edge_list),\" node count:\",len(node_set),\" edge count:\",len(edge_set))\n",
    "#     y_data_4_10.append([loss_avg,labels_4_10[f_path],f_path])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "pred_label_h051={}\n",
    "\n",
    "test_path=\"graph_9_23_h051/\"\n",
    "    \n",
    "filelist = os.listdir(test_path)\n",
    "filelist.sort()\n",
    "for f in filelist:\n",
    "    pred_label_h051[test_path+f]=0\n",
    "    \n",
    "test_path=\"graph_9_24_h051/\"\n",
    "    \n",
    "filelist = os.listdir(test_path)\n",
    "filelist.sort()\n",
    "for f in filelist:\n",
    "    pred_label_h051[test_path+f]=0\n",
    "    \n",
    "test_path=\"graph_9_25_h051/\"\n",
    "    \n",
    "filelist = os.listdir(test_path)\n",
    "filelist.sort()\n",
    "for f in filelist:\n",
    "    pred_label_h051[test_path+f]=0\n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "for hl in history_list_9_25_h051:\n",
    "    loss_count=0\n",
    "    for hq in hl:\n",
    "        if loss_count==0:\n",
    "            loss_count=(loss_count+1)*(hq['loss']+1)\n",
    "        else:\n",
    "            loss_count=(loss_count)*(hq['loss']+1)\n",
    "    name_list=[]\n",
    "    if loss_count>2000:\n",
    "        name_list=[]\n",
    "        for i in hl:\n",
    "            name_list.append(i['name'])\n",
    "\n",
    "        print(*name_list, sep = \"\\n\")\n",
    "        for i in name_list:\n",
    "            pred_label_h051[\"graph_9_25_h051/\"+i]=1\n",
    "        print(loss_count)\n",
    "        \n",
    "        \n",
    "# evaluate\n",
    "y=[]\n",
    "y_pred=[]\n",
    "for i in label_h051:\n",
    "    y.append(label_h051[i])\n",
    "    y_pred.append(pred_label_h051[i])\n",
    "    \n",
    "classifier_evaluation(y,y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## h207"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "\n",
    "node_IDF=torch.load(\"node_IDF_9_22_hosts\")\n",
    "\n",
    "# node_set_list=[]\n",
    "history_list_9_25_h207=[]\n",
    "tw_que=[]\n",
    "his_tw={}\n",
    "current_tw={}\n",
    "\n",
    "test_path=\"graph_9_25_h207/\"\n",
    "\n",
    "\n",
    "file_l=os.listdir(test_path)\n",
    "file_l.sort()\n",
    "index_count=0\n",
    "for f_path in (file_l):\n",
    "    f=open(test_path+f_path)\n",
    "    edge_loss_list=[]\n",
    "    edge_list=[]\n",
    "    print('index_count:',index_count)\n",
    "    \n",
    "    for line in f:\n",
    "        l=line.strip()\n",
    "        jdata=eval(l)\n",
    "        edge_loss_list.append(jdata['loss'])\n",
    "        edge_list.append([str(jdata['srcmsg']).split(\"_@\")[-1],str(jdata['dstmsg']).split(\"_@\")[-1]])\n",
    "#         edge_list.append([str(jdata['srcmsg']),str(jdata['dstmsg'])])\n",
    "        \n",
    "#     df_list_9_22.append(pd.DataFrame(edge_loss_list))\n",
    "    count,loss_avg,node_set,edge_set=cal_anomaly_loss(edge_loss_list,edge_list,test_path)\n",
    "    current_tw['name']=f_path\n",
    "    current_tw['loss']=loss_avg\n",
    "    current_tw['index']=index_count\n",
    "    current_tw['nodeset']=node_set\n",
    "\n",
    "    added_que_flag=False\n",
    "    for hq in history_list_9_25_h207:\n",
    "        for his_tw in hq:\n",
    "            if cal_set_rel(current_tw['nodeset'],his_tw['nodeset'])!=0 and current_tw['name']!=his_tw['name']:\n",
    "                hq.append(copy.deepcopy(current_tw))\n",
    "                print(f\"{his_tw['name']=}\")\n",
    "                added_que_flag=True\n",
    "                break\n",
    "            if added_que_flag:\n",
    "                break\n",
    "    if added_que_flag is False:\n",
    "        temp_hq=[copy.deepcopy(current_tw)]\n",
    "        history_list_9_25_h207.append(temp_hq)\n",
    "  \n",
    "    index_count+=1\n",
    "\n",
    "    print( f_path,\"  \",loss_avg,\" count:\",count,\" percentage:\",count/len(edge_list),\" node count:\",len(node_set),\" edge count:\",len(edge_set))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_label_h207={}\n",
    "\n",
    "test_path=\"graph_9_23_h207/\"\n",
    "    \n",
    "filelist = os.listdir(test_path)\n",
    "filelist.sort()\n",
    "for f in filelist:\n",
    "    pred_label_h207[test_path+f]=0\n",
    "    \n",
    "test_path=\"graph_9_24_h207/\"\n",
    "    \n",
    "filelist = os.listdir(test_path)\n",
    "filelist.sort()\n",
    "for f in filelist:\n",
    "    pred_label_h207[test_path+f]=0\n",
    "    \n",
    "test_path=\"graph_9_25_h207/\"\n",
    "    \n",
    "filelist = os.listdir(test_path)\n",
    "filelist.sort()\n",
    "for f in filelist:\n",
    "    pred_label_h207[test_path+f]=0\n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "for hl in history_list_9_25_h207:\n",
    "    loss_count=0\n",
    "    for hq in hl:\n",
    "        if loss_count==0:\n",
    "            loss_count=(loss_count+1)*(hq['loss']+1)\n",
    "        else:\n",
    "            loss_count=(loss_count)*(hq['loss']+1)\n",
    "    name_list=[]\n",
    "    if loss_count>2000:\n",
    "        name_list=[]\n",
    "        for i in hl:\n",
    "            name_list.append(i['name'])\n",
    "\n",
    "        print(*name_list, sep = \"\\n\")\n",
    "        for i in name_list:\n",
    "            pred_label_h207[\"graph_9_25_h207/\"+i]=1\n",
    "        print(loss_count)\n",
    "        \n",
    "        \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# evaluate\n",
    "y=[]\n",
    "y_pred=[]\n",
    "for i in label_h207:\n",
    "    y.append(label_h207[i])\n",
    "    y_pred.append(pred_label_h207[i])\n",
    "    \n",
    "    \n",
    "classifier_evaluation(y,y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Overall evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "y=[]\n",
    "y_pred=[]\n",
    "\n",
    "for i in label_h201:\n",
    "    y.append(label_h201[i])\n",
    "    y_pred.append(pred_label_h201[i])\n",
    "\n",
    "for i in label_h402:\n",
    "    y.append(label_h402[i])\n",
    "    y_pred.append(pred_label_h402[i])\n",
    "\n",
    "for i in label_h660:\n",
    "    y.append(label_h660[i])\n",
    "    y_pred.append(pred_label_h660[i])\n",
    "\n",
    "for i in label_h501:\n",
    "    y.append(label_h501[i])\n",
    "    y_pred.append(pred_label_h501[i])\n",
    "\n",
    "for i in label_h051:\n",
    "    y.append(label_h051[i])\n",
    "    y_pred.append(pred_label_h051[i])\n",
    "\n",
    "for i in label_h207:\n",
    "    y.append(label_h207[i])\n",
    "    y_pred.append(pred_label_h207[i])\n",
    "    \n",
    "    \n",
    "classifier_evaluation(y,y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "# Attack investigation"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "replace_dic={\n",
    "    '.pyc':'*.pyc',\n",
    "#     '.dll':'*.dll',\n",
    "#     '.DLL':'*.DLL',\n",
    "}\n",
    "\n",
    "def replace_path_name(path_name):\n",
    "    for i in replace_dic:\n",
    "        if i in path_name:\n",
    "            return replace_dic[i]\n",
    "    if '->' in path_name:\n",
    "        if 'outbound' in path_name:\n",
    "            msg=re.findall(\"->(.*?):\",path_name)[0]\n",
    "            return msg\n",
    "        elif 'inbound' in path_name:\n",
    "            msg=re.findall(\"#(.*?):\",path_name)[0]\n",
    "            return msg\n",
    "    return path_name"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Count the number of attack edges"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "label_df=pd.read_csv(\"./labels.csv\")"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### h201"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nodes_attack_h201={}\n",
    "attack_node_set_h201=set()\n",
    "edges_attack_hashset_h201=set()\n",
    "h201_df_label=label_df[label_df['hostname']=='SysClient0201.systemia.com']\n",
    "\n",
    "for idx,row in h201_df_label.iterrows():\n",
    "    \n",
    "    srcflag=False\n",
    "    dstflag=False\n",
    "    if row['objectID'] in node_uuid2path:\n",
    "        nodes_attack_h201[row['objectID']]=replace_path_name(node_uuid2path[row['objectID']])\n",
    "\n",
    "        dstflag=True\n",
    "    if row['actorID'] in node_uuid2path:\n",
    "        nodes_attack_h201[row['actorID']]=replace_path_name(node_uuid2path[row['actorID']])\n",
    "        \n",
    "        srcflag=True\n",
    "    if srcflag and dstflag and row['action'] in rel2id:    \n",
    "        temp_edge=replace_path_name(node_uuid2path[row['actorID']])+','+replace_path_name(node_uuid2path[row['objectID']])+','+str(datetime_to_timestamp_US(row['timestamp']))\n",
    "        edges_attack_hashset_h201.add(hashgen(temp_edge))\n",
    "\n",
    "for i in nodes_attack_h201:\n",
    "    attack_node_set_h201.add(nodes_attack_h201[i])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### h402"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nodes_attack_h402={}\n",
    "attack_node_set_h402=set()\n",
    "edges_attack_hashset_h402=set()\n",
    "h402_df_label=label_df[label_df['hostname']=='SysClient0402.systemia.com']\n",
    "\n",
    "for idx,row in h402_df_label.iterrows():\n",
    "    \n",
    "    srcflag=False\n",
    "    dstflag=False\n",
    "    if row['objectID'] in node_uuid2path:\n",
    "        nodes_attack_h402[row['objectID']]=replace_path_name(node_uuid2path[row['objectID']])\n",
    "        dstflag=True\n",
    "    if row['actorID'] in node_uuid2path:\n",
    "        nodes_attack_h402[row['actorID']]=replace_path_name(node_uuid2path[row['actorID']])\n",
    "        \n",
    "        srcflag=True\n",
    "    if srcflag and dstflag and row['action'] in rel2id:    \n",
    "        temp_edge=replace_path_name(node_uuid2path[row['actorID']])+','+replace_path_name(node_uuid2path[row['objectID']])+','+str(datetime_to_timestamp_US(row['timestamp']))\n",
    "        edges_attack_hashset_h402.add(hashgen(temp_edge))\n",
    "\n",
    "for i in nodes_attack_h402:\n",
    "    attack_node_set_h402.add(nodes_attack_h402[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(edges_attack_hashset_h402)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### h660"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nodes_attack_h660={}\n",
    "attack_node_set_h660=set()\n",
    "edges_attack_hashset_h660=set()\n",
    "h660_df_label=label_df[label_df['hostname']=='SysClient0660.systemia.com']\n",
    "\n",
    "for idx,row in h660_df_label.iterrows():\n",
    "    \n",
    "    srcflag=False\n",
    "    dstflag=False\n",
    "    if row['objectID'] in node_uuid2path:\n",
    "        nodes_attack_h660[row['objectID']]=replace_path_name(node_uuid2path[row['objectID']])\n",
    "        dstflag=True\n",
    "    if row['actorID'] in node_uuid2path:\n",
    "        nodes_attack_h660[row['actorID']]=replace_path_name(node_uuid2path[row['actorID']])\n",
    "        srcflag=True\n",
    "    if srcflag and dstflag and row['action'] in rel2id:    \n",
    "        temp_edge=replace_path_name(node_uuid2path[row['actorID']])+','+replace_path_name(node_uuid2path[row['objectID']])+','+str(datetime_to_timestamp_US(row['timestamp']))\n",
    "        edges_attack_hashset_h660.add(hashgen(temp_edge))\n",
    "\n",
    "for i in nodes_attack_h660:\n",
    "    attack_node_set_h660.add(nodes_attack_h660[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(edges_attack_hashset_h660)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### h501"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nodes_attack_h501={}\n",
    "attack_node_set_h501=set()\n",
    "edges_attack_hashset_h501=set()\n",
    "h501_df_label=label_df[label_df['hostname']=='SysClient0501.systemia.com']\n",
    "\n",
    "for idx,row in h501_df_label.iterrows():\n",
    "    \n",
    "    srcflag=False\n",
    "    dstflag=False\n",
    "    if row['objectID'] in node_uuid2path:\n",
    "        nodes_attack_h501[row['objectID']]=replace_path_name(node_uuid2path[row['objectID']])\n",
    "        dstflag=True\n",
    "    if row['actorID'] in node_uuid2path:\n",
    "        nodes_attack_h501[row['actorID']]=replace_path_name(node_uuid2path[row['actorID']])\n",
    "        srcflag=True\n",
    "    if srcflag and dstflag and row['action'] in rel2id:    \n",
    "        temp_edge=replace_path_name(node_uuid2path[row['actorID']])+','+replace_path_name(node_uuid2path[row['objectID']])+','+str(datetime_to_timestamp_US(row['timestamp']))\n",
    "        edges_attack_hashset_h501.add(hashgen(temp_edge))\n",
    "\n",
    "for i in nodes_attack_h501:\n",
    "    attack_node_set_h501.add(nodes_attack_h501[i])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### h051"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nodes_attack_h051={}\n",
    "attack_node_set_h051=set()\n",
    "edges_attack_hashset_h051=set()\n",
    "h051_df_label=label_df[label_df['hostname']=='SysClient0051.systemia.com']\n",
    "\n",
    "for idx,row in h051_df_label.iterrows():\n",
    "    \n",
    "    srcflag=False\n",
    "    dstflag=False\n",
    "    if row['objectID'] in node_uuid2path:\n",
    "        nodes_attack_h051[row['objectID']]=replace_path_name(node_uuid2path[row['objectID']])\n",
    "        dstflag=True\n",
    "    if row['actorID'] in node_uuid2path:\n",
    "        nodes_attack_h051[row['actorID']]=replace_path_name(node_uuid2path[row['actorID']])\n",
    "        srcflag=True\n",
    "    if srcflag and dstflag and row['action'] in rel2id:    \n",
    "        temp_edge=replace_path_name(node_uuid2path[row['actorID']])+','+replace_path_name(node_uuid2path[row['objectID']])+','+str(datetime_to_timestamp_US(row['timestamp']))\n",
    "        edges_attack_hashset_h051.add(hashgen(temp_edge))\n",
    "\n",
    "for i in nodes_attack_h051:\n",
    "    attack_node_set_h051.add(nodes_attack_h051[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## h201"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "attack_list=[\n",
    "    'graph_9_23_h201/2019-09-23 09:52:00.187~2019-09-23 10:06:52.508.txt',\n",
    "  'graph_9_23_h201/2019-09-23 11:23:44.136~2019-09-23 11:38:30.698.txt',\n",
    " 'graph_9_23_h201/2019-09-23 11:38:40.698~2019-09-23 11:53:39.57.txt',\n",
    " \n",
    " 'graph_9_23_h201/2019-09-23 12:38:24.95~2019-09-23 12:54:14.286.txt',\n",
    " 'graph_9_23_h201/2019-09-23 12:55:28.286~2019-09-23 13:09:50.95.txt',\n",
    " 'graph_9_23_h201/2019-09-23 13:10:24.95~2019-09-23 13:24:56.43.txt',\n",
    "    'graph_9_23_h201/2019-09-23 15:24:48.426~2019-09-23 15:42:02.967.txt',\n",
    "            ]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime, timezone\n",
    "attack_dic=[]\n",
    "for a in attack_list:\n",
    "    temp_dic={}\n",
    "    aa=a.split(\"/\")[-1]\n",
    "    sp=aa.split('~')\n",
    "    start_timestamp=sp[0].split('.')[0]\n",
    "    end_timestamp=sp[1].split('.')[0]\n",
    "    temp_dic['start']=datetime_to_timestamp_US(start_timestamp)\n",
    "    temp_dic['end']=datetime_to_timestamp_US(end_timestamp)\n",
    "    temp_dic['file']=a\n",
    "    attack_dic.append(temp_dic)\n",
    "    print(temp_dic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "\n",
    "original_edges_count=0\n",
    "hash2msg={}\n",
    "graphs=[]\n",
    "gg=nx.DiGraph()\n",
    "count=0\n",
    "# file_list=os.listdir(\"./test_day_data4_10_emb100/\")\n",
    "for path in tqdm(attack_list):\n",
    "#     print(path)\n",
    "    if \".txt\" in path:\n",
    "        line_count=0\n",
    "        node_set=set()\n",
    "        tempg=nx.DiGraph()\n",
    "        f=open(path,\"r\")       \n",
    "        edge_list=[]\n",
    "        for line in f:\n",
    "            count+=1\n",
    "            l=line.strip()\n",
    "            jdata=eval(l)\n",
    "#             temp_key=jdata['srcmsg']+jdata['dstmsg']+jdata['edge_type']\n",
    "#             if temp_key in train_edge_set:\n",
    "#                 jdata['loss']=(jdata['loss']-train_edge_set[temp_key]) if jdata['loss']>=train_edge_set[temp_key] else 0  \n",
    "#             jdata['loss']=abs(jdata['loss']-train_edge_set[temp_key])  if temp_key in train_edge_set else jdata['loss']\n",
    "            edge_list.append(jdata)\n",
    "            \n",
    "        edge_list = sorted(edge_list, key=lambda x:x['loss'],reverse=True) \n",
    "        original_edges_count+=len(edge_list)\n",
    "        \n",
    "        loss_list=[]\n",
    "        for i in edge_list:\n",
    "            loss_list.append(i['loss'])\n",
    "        loss_mean=mean(loss_list)\n",
    "        loss_std=std(loss_list)\n",
    "        print(loss_mean)\n",
    "        print(loss_std)\n",
    "        thr=loss_mean+1.5*loss_std\n",
    "#         thr=-99\n",
    "        print(\"thr:\",thr)\n",
    "        for e in edge_list:\n",
    "            if e['loss']>thr:    \n",
    "#             if True:  \n",
    "#                 if \"'/home/admin/profile'\" in e['srcmsg'] or \" '/home/admin/profile'\" in e['dstmsg']:\n",
    "#                     print(e['srcmsg'])\n",
    "#                     print(e['dstmsg'])\n",
    "                tempg.add_edge(str(hashgen(replace_path_name(e['srcmsg']))),str(hashgen(replace_path_name(e['dstmsg']))))\n",
    "                gg.add_edge(str(hashgen(replace_path_name(e['srcmsg']))),str(hashgen(replace_path_name(e['dstmsg']))),loss=e['loss'],srcmsg=e['srcmsg'],dstmsg=e['dstmsg'],edge_type=e['edge_type'],time=e['time'])\n",
    "                \n",
    "                hash2msg[str(hashgen(replace_path_name(e['srcmsg'])))]=replace_path_name(e['srcmsg'])\n",
    "                hash2msg[str(hashgen(replace_path_name(e['dstmsg'])))]=replace_path_name(e['dstmsg'])\n",
    "\n",
    "#                 gg.add_edge(e['srcnode'],e['dstnode'],loss=e['loss'],srcmsg=e['srcmsg'],dstmsg=e['dstmsg'],edge_type=e['edge_type'],time=e['time'])\n",
    "        print(path)\n",
    "        print(\"tempg edges:\",len(tempg.edges))\n",
    "        print(\"tempg nodes:\",len(tempg.nodes))\n",
    "        print(\"tempg weakly components:\",nx.number_weakly_connected_components(tempg))\n",
    "        \n",
    "        print(\"gg edges:\",len(gg.edges))\n",
    "        print(\"gg nodes:\",len(gg.nodes))\n",
    "        print(\"gg weakly components:\",nx.number_weakly_connected_components(gg))\n",
    "        print(f\"{original_edges_count=}\")\n",
    "#                 gg.add_edge(e['srcnode'],e['dstnode'],loss=e['loss'],srcmsg=e['srcmsg'],dstmsg=e['dstmsg'],edge_type=e['edge_type'],time=e['time'])\n",
    "#         print(path,\" line_count:\",line_count,\"  nodes count:\",len(node_set))\n",
    "#         print(path,\" line_count:\",line_count,\"  nodes count:\",len(node_set))\n",
    "                \n",
    "                \n",
    "                #         graphs.append(g)\n",
    "\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(hash2msg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# compute the best partition\n",
    "import datetime\n",
    "import community as community_louvain\n",
    "starttime = datetime.datetime.now()\n",
    "#long running\n",
    "partition = community_louvain.best_partition(gg.to_undirected())\n",
    "#do something other\n",
    "endtime = datetime.datetime.now()\n",
    "print(\"Finished the computation of community discovery. Execution time:{:d}\".format((endtime - starttime).seconds))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "communities={}\n",
    "max_partition=0\n",
    "for i in partition:\n",
    "    if partition[i]>max_partition:\n",
    "        max_partition=partition[i]\n",
    "        \n",
    "for i in range(max_partition+1):\n",
    "    communities[i]=nx.DiGraph()\n",
    "for e in gg.edges:\n",
    "#     if partition[e[0]]==partition[e[1]]:\n",
    "    communities[partition[e[0]]].add_edge(e[0],e[1])\n",
    "    communities[partition[e[1]]].add_edge(e[0],e[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_partition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def attack_edge_flag(msg):\n",
    "    attack_edge_type=[\n",
    "            '142.20.56.204',\n",
    "        'lsass.exe',\n",
    "        '142.20.61.130',\n",
    "        '132.197.158.98',\n",
    "        'Credentials',\n",
    "    ]\n",
    "    attack_edge_type=attack_node_set_h201\n",
    "    flag=False\n",
    "    for i in attack_edge_type:\n",
    "        if i in msg:\n",
    "            flag=True\n",
    "    return flag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_time_window(edge,tw_dic):\n",
    "    for t in tw_dic:\n",
    "#         print(t['start'])\n",
    "        if edge['time']>=t['start'] and edge['time']<=t['end']:\n",
    "#             print(t['file'])\n",
    "            return t['file']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "max_edge_count=-99\n",
    "max_index=-99\n",
    "max_node_count=0\n",
    "\n",
    "graph_index=0\n",
    "for c in communities:\n",
    "    file_set=set()\n",
    "    if len(communities[c].edges)>max_edge_count:\n",
    "        max_edge_count=len(communities[c].edges)\n",
    "        max_index=graph_index\n",
    "        max_node_count=len(communities[c].nodes)\n",
    "    for e in communities[c].edges:    \n",
    "        try:\n",
    "            temp_edge=gg.edges[e]\n",
    "            file_set.add(find_time_window(temp_edge,attack_dic))\n",
    "        except:\n",
    "            pass    \n",
    "    print(f\"{graph_index=}\")\n",
    "    print(f\"file_set：\")\n",
    "    print(*file_set, sep = \"\\n\")\n",
    "    print(f\"{file_set=}\")\n",
    "    graph_index+=1\n",
    "    \n",
    "print(f\"{max_index=}\")\n",
    "print(f\"{max_edge_count=}\")\n",
    "print(f\"{max_node_count=}\") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_sub_list={'graph_9_23_h201/2019-09-23 12:55:28.286~2019-09-23 13:09:50.95.txt', 'graph_9_23_h201/2019-09-23 13:10:24.95~2019-09-23 13:24:56.43.txt', 'graph_9_23_h201/2019-09-23 12:38:24.95~2019-09-23 12:54:14.286.txt', 'graph_9_23_h201/2019-09-23 11:38:40.698~2019-09-23 11:53:39.57.txt', 'graph_9_23_h201/2019-09-23 11:23:44.136~2019-09-23 11:38:30.698.txt'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "original_edges_count=0\n",
    "\n",
    "for path in tqdm(file_sub_list):\n",
    "\n",
    "    if \".txt\" in path:\n",
    "        line_count=0\n",
    "        node_set=set()\n",
    "        tempg=nx.DiGraph()\n",
    "        f=open(path,\"r\")       \n",
    "        edge_list=[]\n",
    "        for line in f:\n",
    "            edge_list.append(line)\n",
    "\n",
    "        original_edges_count+=len(edge_list)\n",
    "\n",
    "        print(f\"{original_edges_count=}\")\n",
    "\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from graphviz import Digraph\n",
    "\n",
    "\n",
    "graph_index=0\n",
    "\n",
    "total_nodes=0\n",
    "total_edges=0\n",
    "attck_nodes_set_visual=set()\n",
    "for c in communities:\n",
    "    dot = Digraph(name=\"MyPicture\", comment=\"the test\", format=\"pdf\")\n",
    "    dot.graph_attr['rankdir'] = 'LR'\n",
    "    \n",
    "    total_nodes+=len(communities[c].nodes)\n",
    "    total_edges+=len(communities[c].edges)\n",
    "    \n",
    "    subgraph_loss_sum=0\n",
    "    attack_node_count=0\n",
    "    attack_edge_count=0\n",
    "    for e in communities[c].edges:\n",
    "        try:\n",
    "            temp_edge=gg.edges[e]\n",
    "            srcnode=e['srcnode']\n",
    "            dstnode=e['dstnode']\n",
    "        except:\n",
    "            pass        \n",
    "\n",
    "        if True:\n",
    "            subgraph_loss_sum+=temp_edge['loss']\n",
    "   \n",
    "\n",
    "            if \"'subject': '\" in temp_edge['srcmsg']:\n",
    "                src_shape='box'\n",
    "            elif \"'file': '\" in temp_edge['srcmsg']:\n",
    "                src_shape='oval'\n",
    "            elif \"'netflow': '\" in temp_edge['srcmsg']:\n",
    "                src_shape='diamond'\n",
    "                \n",
    "            src_shape='box'\n",
    "            if attack_edge_flag(temp_edge['srcmsg']):\n",
    "                src_node_color='red'\n",
    "                total_nodes+=1\n",
    "            else:\n",
    "                src_node_color='blue'\n",
    "                \n",
    "            src_node_color='blue'\n",
    "            \n",
    "            dot.node( name=str(hashgen(replace_path_name(temp_edge['srcmsg']))),\n",
    "                     label=str(replace_path_name(temp_edge['srcmsg'])+'\\t partition:'+str(partition[str(hashgen(replace_path_name(temp_edge['srcmsg'])))])), \n",
    "                     color=src_node_color,\n",
    "                     shape = src_shape)\n",
    "\n",
    "\n",
    "            if \"'subject': '\" in temp_edge['dstmsg']:\n",
    "                dst_shape='box'\n",
    "            elif \"'file': '\" in temp_edge['dstmsg']:\n",
    "                dst_shape='oval'\n",
    "            elif \"'netflow': '\" in temp_edge['dstmsg']:\n",
    "                dst_shape='diamond'\n",
    "\n",
    "                    \n",
    "            if \"->\" in temp_edge['dstmsg']:\n",
    "                dst_shape='diamond'\n",
    "            else:\n",
    "                dst_shape='oval'\n",
    "                \n",
    "                \n",
    "            if attack_edge_flag(temp_edge['dstmsg']):\n",
    "                dst_node_color='red'\n",
    "                total_nodes+=1\n",
    "            else:\n",
    "                dst_node_color='blue'\n",
    "            dst_node_color='blue'\n",
    "            \n",
    "            \n",
    "            dot.node( name=str(hashgen(replace_path_name(temp_edge['dstmsg']))),\n",
    "                     label=str(replace_path_name(temp_edge['dstmsg'])+'\\t partition:'+str(partition[str(hashgen(replace_path_name(temp_edge['dstmsg'])))])), \n",
    "                     color=dst_node_color,\n",
    "                     shape = dst_shape)\n",
    "\n",
    "\n",
    "    #         edgeindex=tensor_find(test_data.msg[i][16:-16],1)\n",
    "\n",
    "    \n",
    "            temp_edge_visual=replace_path_name(temp_edge['srcmsg'])+','+replace_path_name(temp_edge['dstmsg'])+','+str(temp_edge['time'])\n",
    "            temp_edge_hash_val=hashgen(temp_edge_visual)\n",
    "            \n",
    "            if temp_edge_hash_val in edges_attack_hashset_h201:\n",
    "                edge_color='red'\n",
    "                attck_nodes_set_visual.add(replace_path_name(temp_edge['srcmsg']))\n",
    "                attck_nodes_set_visual.add(replace_path_name(temp_edge['dstmsg']))\n",
    "                attack_edge_count+=1\n",
    "            else:\n",
    "                edge_color='blue'\n",
    "            \n",
    "#             if attack_edge_flag(temp_edge['srcmsg']) and attack_edge_flag(temp_edge['dstmsg']):\n",
    "#                 edge_color='red'\n",
    "#                 attack_edge_count+=1\n",
    "#             else:\n",
    "#                 edge_color='blue'\n",
    "                \n",
    "                \n",
    "            dot.edge(str(hashgen(replace_path_name(temp_edge['srcmsg']))),str(hashgen(replace_path_name(temp_edge['dstmsg']))), label= temp_edge['edge_type'] , color=edge_color)#+ \"  loss: \"+str(temp_edge['loss']) + \"  time: \"+str(temp_edge['time'])\n",
    "\n",
    "    #         dot.edge(str(srcnode), str(dstnode), label= temp_edge['edge_type']+ \"  loss: \"+str((temp_edge['loss'])) + \"  time: \"+str(temp_edge['time']) , color='red')\n",
    "\n",
    "\n",
    "#     if len(communities[c].edges)<2:\n",
    "#         graph_index+=1\n",
    "#         continue\n",
    "#     if len(communities[c].edges)>1000:\n",
    "#         graph_index+=1\n",
    "#         continue\n",
    "#         print(f\"edge num:{len(communities[c].edges)}\")\n",
    "        \n",
    "    print(\"Start to render the figures···\")\n",
    "    \n",
    "    dot.render('./graph_visual_h201/subgraph_'+str(graph_index), view=False)\n",
    "    print(\"subgraph loss:\",(subgraph_loss_sum/len(communities[c].edges)))\n",
    "    print(\"graph_index:\",graph_index)\n",
    "    print(\"edge_count:\",len(communities[c].edges))\n",
    "    print(\"node_count:\",len(communities[c].nodes))\n",
    "    print(f\"{attack_node_count=}\")\n",
    "    print(f\"{attack_edge_count=}\")\n",
    "    graph_index+=1\n",
    "\n",
    "print(f\"avg edges:{total_edges/len(communities)}\")\n",
    "print(f\"avg nodes:{total_nodes/len(communities)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "attck_nodes_set_visual"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(attck_nodes_set_visual)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp_edge_visual"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## h402"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "attack_list=[\n",
    " 'graph_9_23_h402/2019-09-23 13:10:24.429~2019-09-23 13:25:09.374.txt',\n",
    " 'graph_9_23_h402/2019-09-23 13:10:24.429~2019-09-23 13:25:09.374.txt',\n",
    " 'graph_9_23_h402/2019-09-23 13:25:20.374~2019-09-23 13:40:21.268.txt',\n",
    "    'graph_9_23_h402/2019-09-23 13:40:16.268~2019-09-23 13:55:31.310.txt',\n",
    " 'graph_9_23_h402/2019-09-23 13:55:12.31~2019-09-23 14:10:58.200.txt',\n",
    "            ]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime, timezone\n",
    "attack_dic=[]\n",
    "for a in attack_list:\n",
    "    temp_dic={}\n",
    "    aa=a.split(\"/\")[-1]\n",
    "    sp=aa.split('~')\n",
    "    start_timestamp=sp[0].split('.')[0]\n",
    "    end_timestamp=sp[1].split('.')[0]\n",
    "    temp_dic['start']=datetime_to_timestamp_US(start_timestamp)\n",
    "    temp_dic['end']=datetime_to_timestamp_US(end_timestamp)\n",
    "    temp_dic['file']=a\n",
    "    attack_dic.append(temp_dic)\n",
    "    print(temp_dic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "replace_dic={\n",
    "    '.pyc':'*.pyc',\n",
    "    '.dll':'*.dll',\n",
    "    '.DLL':'*.DLL',\n",
    "}\n",
    "\n",
    "def replace_path_name(path_name):\n",
    "    for i in replace_dic:\n",
    "        if i in path_name:\n",
    "            return replace_dic[i]\n",
    "    if '->' in path_name:\n",
    "        if 'outbound' in path_name:\n",
    "            msg=re.findall(\"->(.*?):\",path_name)[0]\n",
    "            return msg\n",
    "        elif 'inbound' in path_name:\n",
    "            msg=re.findall(\"#(.*?):\",path_name)[0]\n",
    "            return msg\n",
    "    return path_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "original_edges_count=0\n",
    "hash2msg={}\n",
    "graphs=[]\n",
    "gg=nx.DiGraph()\n",
    "count=0\n",
    "# file_list=os.listdir(\"./test_day_data4_10_emb100/\")\n",
    "for path in tqdm(attack_list):\n",
    "#     print(path)\n",
    "    if \".txt\" in path:\n",
    "        line_count=0\n",
    "        node_set=set()\n",
    "        tempg=nx.DiGraph()\n",
    "        f=open(path,\"r\")       \n",
    "        edge_list=[]\n",
    "        for line in f:\n",
    "            count+=1\n",
    "            l=line.strip()\n",
    "            jdata=eval(l)\n",
    "#             temp_key=jdata['srcmsg']+jdata['dstmsg']+jdata['edge_type']\n",
    "#             if temp_key in train_edge_set:\n",
    "#                 jdata['loss']=(jdata['loss']-train_edge_set[temp_key]) if jdata['loss']>=train_edge_set[temp_key] else 0  \n",
    "#             jdata['loss']=abs(jdata['loss']-train_edge_set[temp_key])  if temp_key in train_edge_set else jdata['loss']\n",
    "            edge_list.append(jdata)\n",
    "            \n",
    "        edge_list = sorted(edge_list, key=lambda x:x['loss'],reverse=True) \n",
    "        original_edges_count+=len(edge_list)\n",
    "        \n",
    "        loss_list=[]\n",
    "        for i in edge_list:\n",
    "            loss_list.append(i['loss'])\n",
    "        loss_mean=mean(loss_list)\n",
    "        loss_std=std(loss_list)\n",
    "        print(loss_mean)\n",
    "        print(loss_std)\n",
    "        thr=loss_mean+1.5*loss_std\n",
    "#         thr=-99\n",
    "        print(\"thr:\",thr)\n",
    "        for e in edge_list:\n",
    "            if e['loss']>thr:    \n",
    "#             if True:  \n",
    "#                 if \"'/home/admin/profile'\" in e['srcmsg'] or \" '/home/admin/profile'\" in e['dstmsg']:\n",
    "#                     print(e['srcmsg'])\n",
    "#                     print(e['dstmsg'])\n",
    "                tempg.add_edge(str(hashgen(replace_path_name(e['srcmsg']))),str(hashgen(replace_path_name(e['dstmsg']))))\n",
    "                gg.add_edge(str(hashgen(replace_path_name(e['srcmsg']))),str(hashgen(replace_path_name(e['dstmsg']))),loss=e['loss'],srcmsg=e['srcmsg'],dstmsg=e['dstmsg'],edge_type=e['edge_type'],time=e['time'])\n",
    "                \n",
    "                hash2msg[str(hashgen(replace_path_name(e['srcmsg'])))]=replace_path_name(e['srcmsg'])\n",
    "                hash2msg[str(hashgen(replace_path_name(e['dstmsg'])))]=replace_path_name(e['dstmsg'])\n",
    "                \n",
    "\n",
    "#                 gg.add_edge(e['srcnode'],e['dstnode'],loss=e['loss'],srcmsg=e['srcmsg'],dstmsg=e['dstmsg'],edge_type=e['edge_type'],time=e['time'])\n",
    "        print(path)\n",
    "        print(\"tempg edges:\",len(tempg.edges))\n",
    "        print(\"tempg nodes:\",len(tempg.nodes))\n",
    "        print(\"tempg weakly components:\",nx.number_weakly_connected_components(tempg))\n",
    "        \n",
    "        print(\"gg edges:\",len(gg.edges))\n",
    "        print(\"gg nodes:\",len(gg.nodes))\n",
    "        print(\"gg weakly components:\",nx.number_weakly_connected_components(gg))\n",
    "        print(f\"{original_edges_count=}\")\n",
    "#\n",
    "#                 gg.add_edge(e['srcnode'],e['dstnode'],loss=e['loss'],srcmsg=e['srcmsg'],dstmsg=e['dstmsg'],edge_type=e['edge_type'],time=e['time'])\n",
    "#         print(path,\" line_count:\",line_count,\"  nodes count:\",len(node_set))\n",
    "#         print(path,\" line_count:\",line_count,\"  nodes count:\",len(node_set))\n",
    "                \n",
    "                \n",
    "                #         graphs.append(g)\n",
    "\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# compute the best partition\n",
    "import datetime\n",
    "import community as community_louvain\n",
    "starttime = datetime.datetime.now()\n",
    "#long running\n",
    "partition = community_louvain.best_partition(gg.to_undirected())\n",
    "#do something other\n",
    "endtime = datetime.datetime.now()\n",
    "print(\"Finished the computation of community discovery. Execution time:{:d}\".format((endtime - starttime).seconds))\n",
    "\n",
    "\n",
    "communities={}\n",
    "max_partition=0\n",
    "for i in partition:\n",
    "    if partition[i]>max_partition:\n",
    "        max_partition=partition[i]\n",
    "        \n",
    "for i in range(max_partition+1):\n",
    "    communities[i]=nx.DiGraph()\n",
    "for e in gg.edges:\n",
    "#     if partition[e[0]]==partition[e[1]]:\n",
    "    communities[partition[e[0]]].add_edge(e[0],e[1])\n",
    "    communities[partition[e[1]]].add_edge(e[0],e[1])\n",
    "    \n",
    "print(f\"{max_partition=}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "max_edge_count=-99\n",
    "max_index=-99\n",
    "max_node_count=0\n",
    "\n",
    "graph_index=0\n",
    "for c in communities:\n",
    "    file_set=set()\n",
    "    if len(communities[c].edges)>max_edge_count:\n",
    "        max_edge_count=len(communities[c].edges)\n",
    "        max_index=graph_index\n",
    "        max_node_count=len(communities[c].nodes)\n",
    "    for e in communities[c].edges:    \n",
    "        try:\n",
    "            temp_edge=gg.edges[e]\n",
    "            file_set.add(find_time_window(temp_edge,attack_dic))\n",
    "        except:\n",
    "            pass    \n",
    "    print(f\"{graph_index=}\")\n",
    "    print(f\"file_set：\")\n",
    "    print(*file_set, sep = \"\\n\")\n",
    "    print(f\"{file_set=}\")\n",
    "    graph_index+=1\n",
    "    \n",
    "print(f\"{max_index=}\")\n",
    "print(f\"{max_edge_count=}\")\n",
    "print(f\"{max_node_count=}\") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_sub_list={'graph_9_23_h402/2019-09-23 13:25:20.374~2019-09-23 13:40:21.268.txt', 'graph_9_23_h402/2019-09-23 13:10:24.429~2019-09-23 13:25:09.374.txt'}\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "original_edges_count=0\n",
    "\n",
    "for path in tqdm(file_sub_list):\n",
    "\n",
    "    if \".txt\" in path:\n",
    "        line_count=0\n",
    "        node_set=set()\n",
    "        tempg=nx.DiGraph()\n",
    "        f=open(path,\"r\")       \n",
    "        edge_list=[]\n",
    "        for line in f:\n",
    "            edge_list.append(line)\n",
    "\n",
    "        original_edges_count+=len(edge_list)\n",
    "\n",
    "        print(f\"{original_edges_count=}\")\n",
    "#\n",
    "#                 gg.add_edge(e['srcnode'],e['dstnode'],loss=e['loss'],srcmsg=e['srcmsg'],dstmsg=e['dstmsg'],edge_type=e['edge_type'],time=e['time'])\n",
    "#         print(path,\" line_count:\",line_count,\"  nodes count:\",len(node_set))\n",
    "#         print(path,\" line_count:\",line_count,\"  nodes count:\",len(node_set))\n",
    "                \n",
    "                \n",
    "                #         graphs.append(g)\n",
    "\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(communities)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "\n",
    "from graphviz import Digraph\n",
    "\n",
    "\n",
    "graph_index=0\n",
    "\n",
    "total_nodes=0\n",
    "total_edges=0\n",
    "attck_nodes_set_visual=set()\n",
    "for c in communities:\n",
    "\n",
    "    dot = Digraph(name=\"MyPicture\", comment=\"the test\", format=\"pdf\")\n",
    "    dot.graph_attr['rankdir'] = 'LR'\n",
    "    # dot.node(name='a', label='wo', color='purple')\n",
    "    # dot.node(name='b', label='niu', color='purple')\n",
    "    # dot.node(name='c', label='che', color='purple')\n",
    "    \n",
    "    total_nodes+=len(communities[c].nodes)\n",
    "    total_edges+=len(communities[c].edges)\n",
    "    \n",
    "    subgraph_loss_sum=0\n",
    "    attack_node_count=0\n",
    "    attack_edge_count=0\n",
    "    for e in communities[c].edges:\n",
    "        try:\n",
    "            temp_edge=gg.edges[e]\n",
    "            srcnode=e['srcnode']\n",
    "            dstnode=e['dstnode']\n",
    "        except:\n",
    "            pass        \n",
    "\n",
    "        if True:\n",
    "            subgraph_loss_sum+=temp_edge['loss']\n",
    "\n",
    "    #     g.add_edge(srcnode,dstnode,srcmsg=node2msg[indexid2nodeid[srcnode]],dstmsg=node2msg[indexid2nodeid[dstnode]],loss=df['loss'][i])\n",
    "\n",
    "\n",
    "    #         dot.node( name=str(srcnode),label=str(node_index_id2msg_mal[srcnode]), color='purple',shape = 'box')\n",
    "            if \"'subject': '\" in temp_edge['srcmsg']:\n",
    "                src_shape='box'\n",
    "            elif \"'file': '\" in temp_edge['srcmsg']:\n",
    "                src_shape='oval'\n",
    "            elif \"'netflow': '\" in temp_edge['srcmsg']:\n",
    "                src_shape='diamond'\n",
    "                \n",
    "            src_shape='box'\n",
    "            if attack_edge_flag(temp_edge['srcmsg']):\n",
    "                src_node_color='red'\n",
    "                total_nodes+=1\n",
    "            else:\n",
    "                src_node_color='blue'\n",
    "                \n",
    "            src_node_color='blue'\n",
    "            \n",
    "            dot.node( name=str(hashgen(replace_path_name(temp_edge['srcmsg']))),\n",
    "                     label=str(replace_path_name(temp_edge['srcmsg'])+'\\t partition:'+str(partition[str(hashgen(replace_path_name(temp_edge['srcmsg'])))])), \n",
    "                     color=src_node_color,\n",
    "                     shape = src_shape)\n",
    "\n",
    "\n",
    "            if \"'subject': '\" in temp_edge['dstmsg']:\n",
    "                dst_shape='box'\n",
    "            elif \"'file': '\" in temp_edge['dstmsg']:\n",
    "                dst_shape='oval'\n",
    "            elif \"'netflow': '\" in temp_edge['dstmsg']:\n",
    "                dst_shape='diamond'\n",
    "\n",
    "                    \n",
    "            if \"->\" in temp_edge['dstmsg']:\n",
    "                dst_shape='diamond'\n",
    "            else:\n",
    "                dst_shape='oval'\n",
    "                \n",
    "                \n",
    "            if attack_edge_flag(temp_edge['dstmsg']):\n",
    "                dst_node_color='red'\n",
    "                total_nodes+=1\n",
    "            else:\n",
    "                dst_node_color='blue'\n",
    "            dst_node_color='blue'\n",
    "            \n",
    "            \n",
    "            dot.node( name=str(hashgen(replace_path_name(temp_edge['dstmsg']))),\n",
    "                     label=str(replace_path_name(temp_edge['dstmsg'])+'\\t partition:'+str(partition[str(hashgen(replace_path_name(temp_edge['dstmsg'])))])), \n",
    "                     color=dst_node_color,\n",
    "                     shape = dst_shape)\n",
    "\n",
    "\n",
    "    #         edgeindex=tensor_find(test_data.msg[i][16:-16],1)\n",
    "\n",
    "    \n",
    "            temp_edge_visual=replace_path_name(temp_edge['srcmsg'])+','+replace_path_name(temp_edge['dstmsg'])+','+str(temp_edge['time'])\n",
    "            temp_edge_hash_val=hashgen(temp_edge_visual)\n",
    "            \n",
    "            if temp_edge_hash_val in edges_attack_hashset_h402:\n",
    "                edge_color='red'\n",
    "                attck_nodes_set_visual.add(replace_path_name(temp_edge['srcmsg']))\n",
    "                attck_nodes_set_visual.add(replace_path_name(temp_edge['dstmsg']))\n",
    "                attack_edge_count+=1\n",
    "            else:\n",
    "                edge_color='blue'\n",
    "            \n",
    "#             if attack_edge_flag(temp_edge['srcmsg']) and attack_edge_flag(temp_edge['dstmsg']):\n",
    "#                 edge_color='red'\n",
    "#                 attack_edge_count+=1\n",
    "#             else:\n",
    "#                 edge_color='blue'\n",
    "                \n",
    "                \n",
    "            dot.edge(str(hashgen(replace_path_name(temp_edge['srcmsg']))),str(hashgen(replace_path_name(temp_edge['dstmsg']))), label= temp_edge['edge_type'] , color=edge_color)#+ \"  loss: \"+str(temp_edge['loss']) + \"  time: \"+str(temp_edge['time'])\n",
    "\n",
    "    #         dot.edge(str(srcnode), str(dstnode), label= temp_edge['edge_type']+ \"  loss: \"+str((temp_edge['loss'])) + \"  time: \"+str(temp_edge['time']) , color='red')\n",
    "\n",
    "\n",
    "    if len(communities[c].edges)<2:\n",
    "        graph_index+=1\n",
    "        continue\n",
    "    if len(communities[c].edges)>1000:\n",
    "        graph_index+=1\n",
    "        continue\n",
    "        print(f\"edge num:{len(communities[c].edges)}，skip rendering\")\n",
    "        \n",
    "    print(\"Start to reder the figures···\")\n",
    "    \n",
    "    dot.render('./graph_visual_h402/subgraph_'+str(graph_index), view=False)\n",
    "    print(\"subgraph loss:\",(subgraph_loss_sum/len(communities[c].edges)))\n",
    "    print(\"graph_index:\",graph_index)\n",
    "    print(\"edge_count:\",len(communities[c].edges))\n",
    "    print(\"node_count:\",len(communities[c].nodes))\n",
    "    print(f\"{attack_node_count=}\")\n",
    "    print(f\"{attack_edge_count=}\")\n",
    "    graph_index+=1\n",
    "\n",
    "print(f\"avg edges:{total_edges/len(communities)}\")\n",
    "print(f\"avg nodes:{total_nodes/len(communities)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "attck_nodes_set_visual"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## h660"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "attack_list=[\n",
    "    \n",
    "    'graph_9_23_h660/2019-09-23 09:54:08.697~2019-09-23 10:09:11.523.txt',\n",
    "'graph_9_23_h660/2019-09-23 13:27:28.512~2019-09-23 13:42:30.682.txt',\n",
    " 'graph_9_23_h660/2019-09-23 13:42:24.682~2019-09-23 13:57:57.566.txt',\n",
    " 'graph_9_23_h660/2019-09-23 13:57:20.566~2019-09-23 14:12:59.139.txt',\n",
    "            ]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime, timezone\n",
    "attack_dic=[]\n",
    "for a in attack_list:\n",
    "    temp_dic={}\n",
    "    aa=a.split(\"/\")[-1]\n",
    "    sp=aa.split('~')\n",
    "    start_timestamp=sp[0].split('.')[0]\n",
    "    end_timestamp=sp[1].split('.')[0]\n",
    "    temp_dic['start']=datetime_to_timestamp_US(start_timestamp)\n",
    "    temp_dic['end']=datetime_to_timestamp_US(end_timestamp)\n",
    "    temp_dic['file']=a\n",
    "    attack_dic.append(temp_dic)\n",
    "    print(temp_dic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "replace_dic={\n",
    "    '.pyc':'*.pyc',\n",
    "#     '.dll':'*.dll',\n",
    "#     '.DLL':'*.DLL',\n",
    "}\n",
    "\n",
    "def replace_path_name(path_name):\n",
    "    for i in replace_dic:\n",
    "        if i in path_name:\n",
    "            return replace_dic[i]\n",
    "    if '->' in path_name:\n",
    "        if 'outbound' in path_name:\n",
    "            msg=re.findall(\"->(.*?):\",path_name)[0]\n",
    "            return msg\n",
    "        elif 'inbound' in path_name:\n",
    "            msg=re.findall(\"#(.*?):\",path_name)[0]\n",
    "            return msg\n",
    "    return path_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "\n",
    "original_edges_count=0\n",
    "hash2msg={}\n",
    "graphs=[]\n",
    "gg=nx.DiGraph()\n",
    "count=0\n",
    "# file_list=os.listdir(\"./test_day_data4_10_emb100/\")\n",
    "for path in tqdm(attack_list):\n",
    "#     print(path)\n",
    "    if \".txt\" in path:\n",
    "        line_count=0\n",
    "        node_set=set()\n",
    "        tempg=nx.DiGraph()\n",
    "        f=open(path,\"r\")       \n",
    "        edge_list=[]\n",
    "        for line in f:\n",
    "            count+=1\n",
    "            l=line.strip()\n",
    "            jdata=eval(l)\n",
    "#             temp_key=jdata['srcmsg']+jdata['dstmsg']+jdata['edge_type']\n",
    "#             if temp_key in train_edge_set:\n",
    "#                 jdata['loss']=(jdata['loss']-train_edge_set[temp_key]) if jdata['loss']>=train_edge_set[temp_key] else 0  \n",
    "#             jdata['loss']=abs(jdata['loss']-train_edge_set[temp_key])  if temp_key in train_edge_set else jdata['loss']\n",
    "            edge_list.append(jdata)\n",
    "            \n",
    "        edge_list = sorted(edge_list, key=lambda x:x['loss'],reverse=True) \n",
    "        original_edges_count+=len(edge_list)\n",
    "        \n",
    "        loss_list=[]\n",
    "        for i in edge_list:\n",
    "            loss_list.append(i['loss'])\n",
    "        loss_mean=mean(loss_list)\n",
    "        loss_std=std(loss_list)\n",
    "        print(loss_mean)\n",
    "        print(loss_std)\n",
    "        thr=loss_mean+1.5*loss_std\n",
    "#         thr=-99\n",
    "        print(\"thr:\",thr)\n",
    "        for e in edge_list:\n",
    "            if e['loss']>thr:    \n",
    "#             if True:  \n",
    "#                 if \"'/home/admin/profile'\" in e['srcmsg'] or \" '/home/admin/profile'\" in e['dstmsg']:\n",
    "#                     print(e['srcmsg'])\n",
    "#                     print(e['dstmsg'])\n",
    "                tempg.add_edge(str(hashgen(replace_path_name(e['srcmsg']))),str(hashgen(replace_path_name(e['dstmsg']))))\n",
    "                gg.add_edge(str(hashgen(replace_path_name(e['srcmsg']))),str(hashgen(replace_path_name(e['dstmsg']))),loss=e['loss'],srcmsg=e['srcmsg'],dstmsg=e['dstmsg'],edge_type=e['edge_type'],time=e['time'])\n",
    "                \n",
    "                hash2msg[str(hashgen(replace_path_name(e['srcmsg'])))]=replace_path_name(e['srcmsg'])\n",
    "                hash2msg[str(hashgen(replace_path_name(e['dstmsg'])))]=replace_path_name(e['dstmsg'])\n",
    "                \n",
    "\n",
    "#                 gg.add_edge(e['srcnode'],e['dstnode'],loss=e['loss'],srcmsg=e['srcmsg'],dstmsg=e['dstmsg'],edge_type=e['edge_type'],time=e['time'])\n",
    "        print(path)\n",
    "        print(\"tempg edges:\",len(tempg.edges))\n",
    "        print(\"tempg nodes:\",len(tempg.nodes))\n",
    "        print(\"tempg weakly components:\",nx.number_weakly_connected_components(tempg))\n",
    "        \n",
    "        print(\"gg edges:\",len(gg.edges))\n",
    "        print(\"gg nodes:\",len(gg.nodes))\n",
    "        print(\"gg weakly components:\",nx.number_weakly_connected_components(gg))\n",
    "        print(f\"{original_edges_count=}\")\n",
    "#\n",
    "#                 gg.add_edge(e['srcnode'],e['dstnode'],loss=e['loss'],srcmsg=e['srcmsg'],dstmsg=e['dstmsg'],edge_type=e['edge_type'],time=e['time'])\n",
    "#         print(path,\" line_count:\",line_count,\"  nodes count:\",len(node_set))\n",
    "#         print(path,\" line_count:\",line_count,\"  nodes count:\",len(node_set))\n",
    "                \n",
    "                \n",
    "                #         graphs.append(g)\n",
    "\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# compute the best partition\n",
    "import datetime\n",
    "import community as community_louvain\n",
    "starttime = datetime.datetime.now()\n",
    "#long running\n",
    "partition = community_louvain.best_partition(gg.to_undirected())\n",
    "#do something other\n",
    "endtime = datetime.datetime.now()\n",
    "print(\"Finished the computation of community discovery. Execution time:{:d}\".format((endtime - starttime).seconds))\n",
    "\n",
    "\n",
    "communities={}\n",
    "max_partition=0\n",
    "for i in partition:\n",
    "    if partition[i]>max_partition:\n",
    "        max_partition=partition[i]\n",
    "        \n",
    "for i in range(max_partition+1):\n",
    "    communities[i]=nx.DiGraph()\n",
    "for e in gg.edges:\n",
    "#     if partition[e[0]]==partition[e[1]]:\n",
    "    communities[partition[e[0]]].add_edge(e[0],e[1])\n",
    "    communities[partition[e[1]]].add_edge(e[0],e[1])\n",
    "    \n",
    "print(f\"{max_partition=}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "\n",
    "max_edge_count=-99\n",
    "max_index=-99\n",
    "max_node_count=0\n",
    "\n",
    "graph_index=0\n",
    "for c in communities:\n",
    "    file_set=set()\n",
    "    if len(communities[c].edges)>max_edge_count:\n",
    "        max_edge_count=len(communities[c].edges)\n",
    "        max_index=graph_index\n",
    "        max_node_count=len(communities[c].nodes)\n",
    "    for e in communities[c].edges:    \n",
    "        try:\n",
    "            temp_edge=gg.edges[e]\n",
    "            file_set.add(find_time_window(temp_edge,attack_dic))\n",
    "        except:\n",
    "            pass    \n",
    "    print(f\"{graph_index=}\")\n",
    "    print(f\"file_set：\")\n",
    "    print(*file_set, sep = \"\\n\")\n",
    "    print(f\"{file_set=}\")\n",
    "    graph_index+=1\n",
    "    \n",
    "print(f\"{max_index=}\")\n",
    "print(f\"{max_edge_count=}\")\n",
    "print(f\"{max_node_count=}\") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_sub_list={'graph_9_23_h660/2019-09-23 13:27:28.512~2019-09-23 13:42:30.682.txt', 'graph_9_23_h660/2019-09-23 13:42:24.682~2019-09-23 13:57:57.566.txt', 'graph_9_23_h660/2019-09-23 13:57:20.566~2019-09-23 14:12:59.139.txt'}\n",
    "\n",
    "\n",
    "\n",
    "original_edges_count=0\n",
    "\n",
    "for path in tqdm(file_sub_list):\n",
    "\n",
    "    if \".txt\" in path:\n",
    "        line_count=0\n",
    "        node_set=set()\n",
    "        tempg=nx.DiGraph()\n",
    "        f=open(path,\"r\")       \n",
    "        edge_list=[]\n",
    "        for line in f:\n",
    "            edge_list.append(line)\n",
    "\n",
    "        original_edges_count+=len(edge_list)\n",
    "\n",
    "        print(f\"{original_edges_count=}\")\n",
    "#\n",
    "#                 gg.add_edge(e['srcnode'],e['dstnode'],loss=e['loss'],srcmsg=e['srcmsg'],dstmsg=e['dstmsg'],edge_type=e['edge_type'],time=e['time'])\n",
    "#         print(path,\" line_count:\",line_count,\"  nodes count:\",len(node_set))\n",
    "#         print(path,\" line_count:\",line_count,\"  nodes count:\",len(node_set))\n",
    "                \n",
    "                \n",
    "                #         graphs.append(g)\n",
    "\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "\n",
    "from graphviz import Digraph\n",
    "\n",
    "\n",
    "graph_index=0\n",
    "\n",
    "total_nodes=0\n",
    "total_edges=0\n",
    "attck_nodes_set_visual=set()\n",
    "for c in communities:\n",
    "    dot = Digraph(name=\"MyPicture\", comment=\"the test\", format=\"pdf\")\n",
    "    dot.graph_attr['rankdir'] = 'LR'\n",
    "    # dot.node(name='a', label='wo', color='purple')\n",
    "    # dot.node(name='b', label='niu', color='purple')\n",
    "    # dot.node(name='c', label='che', color='purple')\n",
    "    \n",
    "    total_nodes+=len(communities[c].nodes)\n",
    "    total_edges+=len(communities[c].edges)\n",
    "    \n",
    "    subgraph_loss_sum=0\n",
    "    attack_node_count=0\n",
    "    attack_edge_count=0\n",
    "    for e in communities[c].edges:\n",
    "        try:\n",
    "            temp_edge=gg.edges[e]\n",
    "            srcnode=e['srcnode']\n",
    "            dstnode=e['dstnode']\n",
    "        except:\n",
    "            pass        \n",
    "\n",
    "        if True:\n",
    "            subgraph_loss_sum+=temp_edge['loss']\n",
    "\n",
    "    #     g.add_edge(srcnode,dstnode,srcmsg=node2msg[indexid2nodeid[srcnode]],dstmsg=node2msg[indexid2nodeid[dstnode]],loss=df['loss'][i])\n",
    "\n",
    "\n",
    "    #         dot.node( name=str(srcnode),label=str(node_index_id2msg_mal[srcnode]), color='purple',shape = 'box')\n",
    "            if \"'subject': '\" in temp_edge['srcmsg']:\n",
    "                src_shape='box'\n",
    "            elif \"'file': '\" in temp_edge['srcmsg']:\n",
    "                src_shape='oval'\n",
    "            elif \"'netflow': '\" in temp_edge['srcmsg']:\n",
    "                src_shape='diamond'\n",
    "                \n",
    "            src_shape='box'\n",
    "            if attack_edge_flag(temp_edge['srcmsg']):\n",
    "                src_node_color='red'\n",
    "                total_nodes+=1\n",
    "            else:\n",
    "                src_node_color='blue'\n",
    "                \n",
    "            src_node_color='blue'\n",
    "            \n",
    "            dot.node( name=str(hashgen(replace_path_name(temp_edge['srcmsg']))),\n",
    "                     label=str(replace_path_name(temp_edge['srcmsg'])+'\\t partition:'+str(partition[str(hashgen(replace_path_name(temp_edge['srcmsg'])))])), \n",
    "                     color=src_node_color,\n",
    "                     shape = src_shape)\n",
    "\n",
    "\n",
    "            if \"'subject': '\" in temp_edge['dstmsg']:\n",
    "                dst_shape='box'\n",
    "            elif \"'file': '\" in temp_edge['dstmsg']:\n",
    "                dst_shape='oval'\n",
    "            elif \"'netflow': '\" in temp_edge['dstmsg']:\n",
    "                dst_shape='diamond'\n",
    "\n",
    "                    \n",
    "            if \"->\" in temp_edge['dstmsg']:\n",
    "                dst_shape='diamond'\n",
    "            else:\n",
    "                dst_shape='oval'\n",
    "                \n",
    "                \n",
    "            if attack_edge_flag(temp_edge['dstmsg']):\n",
    "                dst_node_color='red'\n",
    "                total_nodes+=1\n",
    "            else:\n",
    "                dst_node_color='blue'\n",
    "            dst_node_color='blue'\n",
    "            \n",
    "            \n",
    "            dot.node( name=str(hashgen(replace_path_name(temp_edge['dstmsg']))),\n",
    "                     label=str(replace_path_name(temp_edge['dstmsg'])+'\\t partition:'+str(partition[str(hashgen(replace_path_name(temp_edge['dstmsg'])))])), \n",
    "                     color=dst_node_color,\n",
    "                     shape = dst_shape)\n",
    "\n",
    "\n",
    "    #         edgeindex=tensor_find(test_data.msg[i][16:-16],1)\n",
    "\n",
    "    \n",
    "            temp_edge_visual=replace_path_name(temp_edge['srcmsg'])+','+replace_path_name(temp_edge['dstmsg'])+','+str(temp_edge['time'])\n",
    "            temp_edge_hash_val=hashgen(temp_edge_visual)\n",
    "            \n",
    "            if temp_edge_hash_val in edges_attack_hashset_h660:\n",
    "                edge_color='red'\n",
    "                attck_nodes_set_visual.add(replace_path_name(temp_edge['srcmsg']))\n",
    "                attck_nodes_set_visual.add(replace_path_name(temp_edge['dstmsg']))\n",
    "                attack_edge_count+=1\n",
    "            else:\n",
    "                edge_color='blue'\n",
    "            \n",
    "#             if attack_edge_flag(temp_edge['srcmsg']) and attack_edge_flag(temp_edge['dstmsg']):\n",
    "#                 edge_color='red'\n",
    "#                 attack_edge_count+=1\n",
    "#             else:\n",
    "#                 edge_color='blue'\n",
    "                \n",
    "                \n",
    "            dot.edge(str(hashgen(replace_path_name(temp_edge['srcmsg']))),str(hashgen(replace_path_name(temp_edge['dstmsg']))), label= temp_edge['edge_type'] , color=edge_color)#+ \"  loss: \"+str(temp_edge['loss']) + \"  time: \"+str(temp_edge['time'])\n",
    "\n",
    "    #         dot.edge(str(srcnode), str(dstnode), label= temp_edge['edge_type']+ \"  loss: \"+str((temp_edge['loss'])) + \"  time: \"+str(temp_edge['time']) , color='red')\n",
    "\n",
    "\n",
    "#     if len(communities[c].edges)<2:\n",
    "#         continue\n",
    "#     if len(communities[c].edges)>1000:\n",
    "#         continue\n",
    "#         print(f\"edge num:{len(communities[c].edges)}，skip rendering\")\n",
    "        \n",
    "    print(\"Start to render the figures···\")\n",
    "    \n",
    "    dot.render('./graph_visual_h660/subgraph_'+str(graph_index), view=False)\n",
    "    print(\"subgraph loss:\",(subgraph_loss_sum/len(communities[c].edges)))\n",
    "    print(\"graph_index:\",graph_index)\n",
    "    print(\"edge_count:\",len(communities[c].edges))\n",
    "    print(\"node_count:\",len(communities[c].nodes))\n",
    "    print(f\"{attack_node_count=}\")\n",
    "    print(f\"{attack_edge_count=}\")\n",
    "    graph_index+=1\n",
    "\n",
    "print(f\"avg edges:{total_edges/len(communities)}\")\n",
    "print(f\"avg nodes:{total_nodes/len(communities)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "attck_nodes_set_visual"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## h501"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "attack_list=[\n",
    " 'graph_9_24_h501/2019-09-24 10:15:28.241~2019-09-24 10:30:00.201.txt',\n",
    " 'graph_9_24_h501/2019-09-24 10:30:24.201~2019-09-24 10:45:02.7.txt',\n",
    " 'graph_9_24_h501/2019-09-24 10:45:20.7~2019-09-24 11:00:31.385.txt',\n",
    " 'graph_9_24_h501/2019-09-24 11:00:16.385~2019-09-24 11:16:09.755.txt',\n",
    " 'graph_9_24_h501/2019-09-24 11:15:12.755~2019-09-24 11:31:14.287.txt',\n",
    " 'graph_9_24_h501/2019-09-24 11:32:16.287~2019-09-24 11:46:31.541.txt',\n",
    "    \n",
    " 'graph_9_24_h501/2019-09-24 13:04:00.804~2019-09-24 13:17:29.451.txt',\n",
    " 'graph_9_24_h501/2019-09-24 13:18:56.451~2019-09-24 13:32:46.454.txt',\n",
    " 'graph_9_24_h501/2019-09-24 13:33:52.454~2019-09-24 13:48:02.493.txt',\n",
    "            ]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime, timezone\n",
    "attack_dic=[]\n",
    "for a in attack_list:\n",
    "    temp_dic={}\n",
    "    aa=a.split(\"/\")[-1]\n",
    "    sp=aa.split('~')\n",
    "    start_timestamp=sp[0].split('.')[0]\n",
    "    end_timestamp=sp[1].split('.')[0]\n",
    "    temp_dic['start']=datetime_to_timestamp_US(start_timestamp)\n",
    "    temp_dic['end']=datetime_to_timestamp_US(end_timestamp)\n",
    "    temp_dic['file']=a\n",
    "    attack_dic.append(temp_dic)\n",
    "    print(temp_dic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "replace_dic={\n",
    "    '.pyc':'*.pyc',\n",
    "    '.dll':'*.dll',\n",
    "    '.DLL':'*.DLL',\n",
    "}\n",
    "\n",
    "def replace_path_name(path_name):\n",
    "    for i in replace_dic:\n",
    "        if i in path_name:\n",
    "            return replace_dic[i]\n",
    "    if '->' in path_name:\n",
    "        if 'outbound' in path_name:\n",
    "            msg=re.findall(\"->(.*?):\",path_name)[0]\n",
    "            return msg\n",
    "        elif 'inbound' in path_name:\n",
    "            msg=re.findall(\"#(.*?):\",path_name)[0]\n",
    "            return msg\n",
    "    return path_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "\n",
    "original_edges_count=0\n",
    "hash2msg={}\n",
    "graphs=[]\n",
    "gg=nx.DiGraph()\n",
    "count=0\n",
    "# file_list=os.listdir(\"./test_day_data4_10_emb100/\")\n",
    "for path in tqdm(attack_list):\n",
    "#     print(path)\n",
    "    if \".txt\" in path:\n",
    "        line_count=0\n",
    "        node_set=set()\n",
    "        tempg=nx.DiGraph()\n",
    "        f=open(path,\"r\")       \n",
    "        edge_list=[]\n",
    "        for line in f:\n",
    "            count+=1\n",
    "            l=line.strip()\n",
    "            jdata=eval(l)\n",
    "#             temp_key=jdata['srcmsg']+jdata['dstmsg']+jdata['edge_type']\n",
    "#             if temp_key in train_edge_set:\n",
    "#                 jdata['loss']=(jdata['loss']-train_edge_set[temp_key]) if jdata['loss']>=train_edge_set[temp_key] else 0  \n",
    "#             jdata['loss']=abs(jdata['loss']-train_edge_set[temp_key])  if temp_key in train_edge_set else jdata['loss']\n",
    "            edge_list.append(jdata)\n",
    "            \n",
    "        edge_list = sorted(edge_list, key=lambda x:x['loss'],reverse=True) \n",
    "        original_edges_count+=len(edge_list)\n",
    "        \n",
    "        loss_list=[]\n",
    "        for i in edge_list:\n",
    "            loss_list.append(i['loss'])\n",
    "        loss_mean=mean(loss_list)\n",
    "        loss_std=std(loss_list)\n",
    "        print(loss_mean)\n",
    "        print(loss_std)\n",
    "        thr=loss_mean+2.5*loss_std\n",
    "#         thr=-99\n",
    "        print(\"thr:\",thr)\n",
    "        for e in edge_list:\n",
    "            if e['loss']>thr:    \n",
    "#             if True:  \n",
    "#                 if \"'/home/admin/profile'\" in e['srcmsg'] or \" '/home/admin/profile'\" in e['dstmsg']:\n",
    "#                     print(e['srcmsg'])\n",
    "#                     print(e['dstmsg'])\n",
    "                tempg.add_edge(str(hashgen(replace_path_name(e['srcmsg']))),str(hashgen(replace_path_name(e['dstmsg']))))\n",
    "                gg.add_edge(str(hashgen(replace_path_name(e['srcmsg']))),str(hashgen(replace_path_name(e['dstmsg']))),loss=e['loss'],srcmsg=e['srcmsg'],dstmsg=e['dstmsg'],edge_type=e['edge_type'],time=e['time'])\n",
    "                \n",
    "                hash2msg[str(hashgen(replace_path_name(e['srcmsg'])))]=replace_path_name(e['srcmsg'])\n",
    "                hash2msg[str(hashgen(replace_path_name(e['dstmsg'])))]=replace_path_name(e['dstmsg'])\n",
    "                \n",
    "\n",
    "#                 gg.add_edge(e['srcnode'],e['dstnode'],loss=e['loss'],srcmsg=e['srcmsg'],dstmsg=e['dstmsg'],edge_type=e['edge_type'],time=e['time'])\n",
    "        print(path)\n",
    "        print(\"tempg edges:\",len(tempg.edges))\n",
    "        print(\"tempg nodes:\",len(tempg.nodes))\n",
    "        print(\"tempg weakly components:\",nx.number_weakly_connected_components(tempg))\n",
    "        \n",
    "        print(\"gg edges:\",len(gg.edges))\n",
    "        print(\"gg nodes:\",len(gg.nodes))\n",
    "        print(\"gg weakly components:\",nx.number_weakly_connected_components(gg))\n",
    "        print(f\"{original_edges_count=}\")\n",
    "#\n",
    "#                 gg.add_edge(e['srcnode'],e['dstnode'],loss=e['loss'],srcmsg=e['srcmsg'],dstmsg=e['dstmsg'],edge_type=e['edge_type'],time=e['time'])\n",
    "#         print(path,\" line_count:\",line_count,\"  nodes count:\",len(node_set))\n",
    "#         print(path,\" line_count:\",line_count,\"  nodes count:\",len(node_set))\n",
    "                \n",
    "                \n",
    "                #         graphs.append(g)\n",
    "\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# compute the best partition\n",
    "import datetime\n",
    "import community as community_louvain\n",
    "starttime = datetime.datetime.now()\n",
    "#long running\n",
    "partition = community_louvain.best_partition(gg.to_undirected())\n",
    "#do something other\n",
    "endtime = datetime.datetime.now()\n",
    "print(\"Finished the computation of community discovery. Execution time:{:d}\".format((endtime - starttime).seconds))\n",
    "\n",
    "\n",
    "communities={}\n",
    "max_partition=0\n",
    "for i in partition:\n",
    "    if partition[i]>max_partition:\n",
    "        max_partition=partition[i]\n",
    "        \n",
    "for i in range(max_partition+1):\n",
    "    communities[i]=nx.DiGraph()\n",
    "for e in gg.edges:\n",
    "#     if partition[e[0]]==partition[e[1]]:\n",
    "    communities[partition[e[0]]].add_edge(e[0],e[1])\n",
    "    communities[partition[e[1]]].add_edge(e[0],e[1])\n",
    "    \n",
    "print(f\"{max_partition=}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_edge_count=-99\n",
    "max_index=-99\n",
    "max_node_count=0\n",
    "\n",
    "graph_index=0\n",
    "for c in communities:\n",
    "    file_set=set()\n",
    "    if len(communities[c].edges)>max_edge_count:\n",
    "        max_edge_count=len(communities[c].edges)\n",
    "        max_index=graph_index\n",
    "        max_node_count=len(communities[c].nodes)\n",
    "    for e in communities[c].edges:    \n",
    "        try:\n",
    "            temp_edge=gg.edges[e]\n",
    "            file_set.add(find_time_window(temp_edge,attack_dic))\n",
    "        except:\n",
    "            pass    \n",
    "    print(f\"{graph_index=}\")\n",
    "    print(f\"file_set：\")\n",
    "    print(*file_set, sep = \"\\n\")\n",
    "    print(f\"{file_set=}\")\n",
    "    graph_index+=1\n",
    "    \n",
    "print(f\"{max_index=}\")\n",
    "print(f\"{max_edge_count=}\")\n",
    "print(f\"{max_node_count=}\") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_sub_list={'graph_9_24_h501/2019-09-24 10:45:20.7~2019-09-24 11:00:31.385.txt', 'graph_9_24_h501/2019-09-24 11:00:16.385~2019-09-24 11:16:09.755.txt', 'graph_9_24_h501/2019-09-24 10:15:28.241~2019-09-24 10:30:00.201.txt', 'graph_9_24_h501/2019-09-24 11:15:12.755~2019-09-24 11:31:14.287.txt', 'graph_9_24_h501/2019-09-24 11:32:16.287~2019-09-24 11:46:31.541.txt', 'graph_9_24_h501/2019-09-24 10:30:24.201~2019-09-24 10:45:02.7.txt','graph_9_24_h501/2019-09-24 10:15:28.241~2019-09-24 10:30:00.201.txt', 'graph_9_24_h501/2019-09-24 13:04:00.804~2019-09-24 13:17:29.451.txt', 'graph_9_24_h501/2019-09-24 11:15:12.755~2019-09-24 11:31:14.287.txt', 'graph_9_24_h501/2019-09-24 11:32:16.287~2019-09-24 11:46:31.541.txt', 'graph_9_24_h501/2019-09-24 13:33:52.454~2019-09-24 13:48:02.493.txt', 'graph_9_24_h501/2019-09-24 10:30:24.201~2019-09-24 10:45:02.7.txt', 'graph_9_24_h501/2019-09-24 13:18:56.451~2019-09-24 13:32:46.454.txt'}\n",
    "\n",
    "original_edges_count=0\n",
    "\n",
    "for path in tqdm(file_sub_list):\n",
    "\n",
    "    if \".txt\" in path:\n",
    "        line_count=0\n",
    "        node_set=set()\n",
    "        tempg=nx.DiGraph()\n",
    "        f=open(path,\"r\")       \n",
    "        edge_list=[]\n",
    "        for line in f:\n",
    "            edge_list.append(line)\n",
    "\n",
    "        original_edges_count+=len(edge_list)\n",
    "\n",
    "        print(f\"{original_edges_count=}\")\n",
    "#\n",
    "#                 gg.add_edge(e['srcnode'],e['dstnode'],loss=e['loss'],srcmsg=e['srcmsg'],dstmsg=e['dstmsg'],edge_type=e['edge_type'],time=e['time'])\n",
    "#         print(path,\" line_count:\",line_count,\"  nodes count:\",len(node_set))\n",
    "#         print(path,\" line_count:\",line_count,\"  nodes count:\",len(node_set))\n",
    "                \n",
    "                \n",
    "                #         graphs.append(g)\n",
    "\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from graphviz import Digraph\n",
    "\n",
    "\n",
    "graph_index=0\n",
    "\n",
    "total_nodes=0\n",
    "total_edges=0\n",
    "attck_nodes_set_visual=set()\n",
    "for c in communities:\n",
    "    dot = Digraph(name=\"MyPicture\", comment=\"the test\", format=\"pdf\")\n",
    "    dot.graph_attr['rankdir'] = 'LR'\n",
    "    # dot.node(name='a', label='wo', color='purple')\n",
    "    # dot.node(name='b', label='niu', color='purple')\n",
    "    # dot.node(name='c', label='che', color='purple')\n",
    "    \n",
    "    total_nodes+=len(communities[c].nodes)\n",
    "    total_edges+=len(communities[c].edges)\n",
    "    \n",
    "    subgraph_loss_sum=0\n",
    "    attack_node_count=0\n",
    "    attack_edge_count=0\n",
    "    for e in communities[c].edges:\n",
    "        try:\n",
    "            temp_edge=gg.edges[e]\n",
    "            srcnode=e['srcnode']\n",
    "            dstnode=e['dstnode']\n",
    "        except:\n",
    "            pass        \n",
    "\n",
    "        if True:\n",
    "            subgraph_loss_sum+=temp_edge['loss']\n",
    "\n",
    "    #     g.add_edge(srcnode,dstnode,srcmsg=node2msg[indexid2nodeid[srcnode]],dstmsg=node2msg[indexid2nodeid[dstnode]],loss=df['loss'][i])\n",
    "\n",
    "\n",
    "    #         dot.node( name=str(srcnode),label=str(node_index_id2msg_mal[srcnode]), color='purple',shape = 'box')\n",
    "            if \"'subject': '\" in temp_edge['srcmsg']:\n",
    "                src_shape='box'\n",
    "            elif \"'file': '\" in temp_edge['srcmsg']:\n",
    "                src_shape='oval'\n",
    "            elif \"'netflow': '\" in temp_edge['srcmsg']:\n",
    "                src_shape='diamond'\n",
    "                \n",
    "            src_shape='box'\n",
    "            if attack_edge_flag(temp_edge['srcmsg']):\n",
    "                src_node_color='red'\n",
    "                total_nodes+=1\n",
    "            else:\n",
    "                src_node_color='blue'\n",
    "                \n",
    "            src_node_color='blue'\n",
    "            \n",
    "            dot.node( name=str(hashgen(replace_path_name(temp_edge['srcmsg']))),\n",
    "                     label=str(replace_path_name(temp_edge['srcmsg'])+'\\t partition:'+str(partition[str(hashgen(replace_path_name(temp_edge['srcmsg'])))])), \n",
    "                     color=src_node_color,\n",
    "                     shape = src_shape)\n",
    "\n",
    "\n",
    "            if \"'subject': '\" in temp_edge['dstmsg']:\n",
    "                dst_shape='box'\n",
    "            elif \"'file': '\" in temp_edge['dstmsg']:\n",
    "                dst_shape='oval'\n",
    "            elif \"'netflow': '\" in temp_edge['dstmsg']:\n",
    "                dst_shape='diamond'\n",
    "\n",
    "                    \n",
    "            if \"->\" in temp_edge['dstmsg']:\n",
    "                dst_shape='diamond'\n",
    "            else:\n",
    "                dst_shape='oval'\n",
    "                \n",
    "                \n",
    "            if attack_edge_flag(temp_edge['dstmsg']):\n",
    "                dst_node_color='red'\n",
    "                total_nodes+=1\n",
    "            else:\n",
    "                dst_node_color='blue'\n",
    "            dst_node_color='blue'\n",
    "            \n",
    "            \n",
    "            dot.node( name=str(hashgen(replace_path_name(temp_edge['dstmsg']))),\n",
    "                     label=str(replace_path_name(temp_edge['dstmsg'])+'\\t partition:'+str(partition[str(hashgen(replace_path_name(temp_edge['dstmsg'])))])), \n",
    "                     color=dst_node_color,\n",
    "                     shape = dst_shape)\n",
    "\n",
    "\n",
    "    #         edgeindex=tensor_find(test_data.msg[i][16:-16],1)\n",
    "    \n",
    "            temp_edge_visual=replace_path_name(temp_edge['srcmsg'])+','+replace_path_name(temp_edge['dstmsg'])+','+str(temp_edge['time'])\n",
    "            temp_edge_hash_val=hashgen(temp_edge_visual)\n",
    "            \n",
    "            if temp_edge_hash_val in edges_attack_hashset_h501:\n",
    "                edge_color='red'\n",
    "                attck_nodes_set_visual.add(replace_path_name(temp_edge['srcmsg']))\n",
    "                attck_nodes_set_visual.add(replace_path_name(temp_edge['dstmsg']))\n",
    "                attack_edge_count+=1\n",
    "            else:\n",
    "                edge_color='blue'\n",
    "            \n",
    "#             if attack_edge_flag(temp_edge['srcmsg']) and attack_edge_flag(temp_edge['dstmsg']):\n",
    "#                 edge_color='red'\n",
    "#                 attack_edge_count+=1\n",
    "#             else:\n",
    "#                 edge_color='blue'\n",
    "                \n",
    "                \n",
    "            dot.edge(str(hashgen(replace_path_name(temp_edge['srcmsg']))),str(hashgen(replace_path_name(temp_edge['dstmsg']))), label= temp_edge['edge_type'] , color=edge_color)#+ \"  loss: \"+str(temp_edge['loss']) + \"  time: \"+str(temp_edge['time'])\n",
    "\n",
    "    #         dot.edge(str(srcnode), str(dstnode), label= temp_edge['edge_type']+ \"  loss: \"+str((temp_edge['loss'])) + \"  time: \"+str(temp_edge['time']) , color='red')\n",
    "\n",
    "\n",
    "#     if len(communities[c].edges)<2:\n",
    "#         continue\n",
    "#     if len(communities[c].edges)>1000:\n",
    "#         continue\n",
    "#         print(f\"edge num:{len(communities[c].edges)}，skip rendering\")\n",
    "        \n",
    "    print(\"Start to render the figures···\")\n",
    "    \n",
    "    dot.render('./graph_visual_h501/subgraph_'+str(graph_index), view=False)\n",
    "    print(\"subgraph loss:\",(subgraph_loss_sum/len(communities[c].edges)))\n",
    "    print(\"graph_index:\",graph_index)\n",
    "    print(\"edge_count:\",len(communities[c].edges))\n",
    "    print(\"node_count:\",len(communities[c].nodes))\n",
    "    print(f\"{attack_node_count=}\")\n",
    "    print(f\"{attack_edge_count=}\")\n",
    "    graph_index+=1\n",
    "\n",
    "print(f\"avg edges:{total_edges/len(communities)}\")\n",
    "print(f\"avg nodes:{total_nodes/len(communities)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "attck_nodes_set_visual"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## h051"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "attack_list=[\n",
    "    'graph_9_25_h051/2019-09-25 09:11:28.846~2019-09-25 09:25:34.393.txt',\n",
    "    'graph_9_25_h051/2019-09-25 09:26:24.393~2019-09-25 09:40:36.393.txt',\n",
    "'graph_9_25_h051/2019-09-25 10:26:08.397~2019-09-25 10:41:40.247.txt',\n",
    " 'graph_9_25_h051/2019-09-25 10:41:04.247~2019-09-25 10:56:56.92.txt',\n",
    " 'graph_9_25_h051/2019-09-25 10:56:00.92~2019-09-25 11:12:03.608.txt',\n",
    "    'graph_9_25_h051/2019-09-25 14:16:32.762~2019-09-25 14:32:02.764.txt',\n",
    "            ]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime, timezone\n",
    "attack_dic=[]\n",
    "for a in attack_list:\n",
    "    temp_dic={}\n",
    "    aa=a.split(\"/\")[-1]\n",
    "    sp=aa.split('~')\n",
    "    start_timestamp=sp[0].split('.')[0]\n",
    "    end_timestamp=sp[1].split('.')[0]\n",
    "    temp_dic['start']=datetime_to_timestamp_US(start_timestamp)\n",
    "    temp_dic['end']=datetime_to_timestamp_US(end_timestamp)\n",
    "    temp_dic['file']=a\n",
    "    attack_dic.append(temp_dic)\n",
    "    print(temp_dic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "replace_dic={\n",
    "    '.pyc':'*.pyc',\n",
    "    '.dll':'*.dll',\n",
    "    '.DLL':'*.DLL',\n",
    "}\n",
    "\n",
    "def replace_path_name(path_name):\n",
    "    for i in replace_dic:\n",
    "        if i in path_name:\n",
    "            return replace_dic[i]\n",
    "    if '->' in path_name:\n",
    "        if 'outbound' in path_name:\n",
    "            msg=re.findall(\"->(.*?):\",path_name)[0]\n",
    "            return msg\n",
    "        elif 'inbound' in path_name:\n",
    "            msg=re.findall(\"#(.*?):\",path_name)[0]\n",
    "            return msg\n",
    "    return path_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "original_edges_count=0\n",
    "hash2msg={}\n",
    "graphs=[]\n",
    "gg=nx.DiGraph()\n",
    "count=0\n",
    "# file_list=os.listdir(\"./test_day_data4_10_emb100/\")\n",
    "for path in tqdm(attack_list):\n",
    "#     print(path)\n",
    "    if \".txt\" in path:\n",
    "        line_count=0\n",
    "        node_set=set()\n",
    "        tempg=nx.DiGraph()\n",
    "        f=open(path,\"r\")       \n",
    "        edge_list=[]\n",
    "        for line in f:\n",
    "            count+=1\n",
    "            l=line.strip()\n",
    "            jdata=eval(l)\n",
    "#             temp_key=jdata['srcmsg']+jdata['dstmsg']+jdata['edge_type']\n",
    "#             if temp_key in train_edge_set:\n",
    "#                 jdata['loss']=(jdata['loss']-train_edge_set[temp_key]) if jdata['loss']>=train_edge_set[temp_key] else 0  \n",
    "#             jdata['loss']=abs(jdata['loss']-train_edge_set[temp_key])  if temp_key in train_edge_set else jdata['loss']\n",
    "            edge_list.append(jdata)\n",
    "            \n",
    "        edge_list = sorted(edge_list, key=lambda x:x['loss'],reverse=True) \n",
    "        original_edges_count+=len(edge_list)\n",
    "        \n",
    "        loss_list=[]\n",
    "        for i in edge_list:\n",
    "            loss_list.append(i['loss'])\n",
    "        loss_mean=mean(loss_list)\n",
    "        loss_std=std(loss_list)\n",
    "        print(loss_mean)\n",
    "        print(loss_std)\n",
    "        thr=loss_mean+2.5*loss_std\n",
    "#         thr=-99\n",
    "        print(\"thr:\",thr)\n",
    "        for e in edge_list:\n",
    "            if e['loss']>thr:    \n",
    "#             if True:  \n",
    "#                 if \"'/home/admin/profile'\" in e['srcmsg'] or \" '/home/admin/profile'\" in e['dstmsg']:\n",
    "#                     print(e['srcmsg'])\n",
    "#                     print(e['dstmsg'])\n",
    "                tempg.add_edge(str(hashgen(replace_path_name(e['srcmsg']))),str(hashgen(replace_path_name(e['dstmsg']))))\n",
    "                gg.add_edge(str(hashgen(replace_path_name(e['srcmsg']))),str(hashgen(replace_path_name(e['dstmsg']))),loss=e['loss'],srcmsg=e['srcmsg'],dstmsg=e['dstmsg'],edge_type=e['edge_type'],time=e['time'])\n",
    "                \n",
    "                hash2msg[str(hashgen(replace_path_name(e['srcmsg'])))]=replace_path_name(e['srcmsg'])\n",
    "                hash2msg[str(hashgen(replace_path_name(e['dstmsg'])))]=replace_path_name(e['dstmsg'])\n",
    "                \n",
    "\n",
    "#                 gg.add_edge(e['srcnode'],e['dstnode'],loss=e['loss'],srcmsg=e['srcmsg'],dstmsg=e['dstmsg'],edge_type=e['edge_type'],time=e['time'])\n",
    "        print(path)\n",
    "        print(\"tempg edges:\",len(tempg.edges))\n",
    "        print(\"tempg nodes:\",len(tempg.nodes))\n",
    "        print(\"tempg weakly components:\",nx.number_weakly_connected_components(tempg))\n",
    "        \n",
    "        print(\"gg edges:\",len(gg.edges))\n",
    "        print(\"gg nodes:\",len(gg.nodes))\n",
    "        print(\"gg weakly components:\",nx.number_weakly_connected_components(gg))\n",
    "        print(f\"{original_edges_count=}\")\n",
    "#\n",
    "#                 gg.add_edge(e['srcnode'],e['dstnode'],loss=e['loss'],srcmsg=e['srcmsg'],dstmsg=e['dstmsg'],edge_type=e['edge_type'],time=e['time'])\n",
    "#         print(path,\" line_count:\",line_count,\"  nodes count:\",len(node_set))\n",
    "#         print(path,\" line_count:\",line_count,\"  nodes count:\",len(node_set))\n",
    "                \n",
    "                \n",
    "                #         graphs.append(g)\n",
    "\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# compute the best partition\n",
    "import datetime\n",
    "import community as community_louvain\n",
    "starttime = datetime.datetime.now()\n",
    "#long running\n",
    "partition = community_louvain.best_partition(gg.to_undirected())\n",
    "#do something other\n",
    "endtime = datetime.datetime.now()\n",
    "print(\"Finished the computation of community discovery. Execution time:{:d}\".format((endtime - starttime).seconds))\n",
    "\n",
    "\n",
    "communities={}\n",
    "max_partition=0\n",
    "for i in partition:\n",
    "    if partition[i]>max_partition:\n",
    "        max_partition=partition[i]\n",
    "        \n",
    "for i in range(max_partition+1):\n",
    "    communities[i]=nx.DiGraph()\n",
    "for e in gg.edges:\n",
    "#     if partition[e[0]]==partition[e[1]]:\n",
    "    communities[partition[e[0]]].add_edge(e[0],e[1])\n",
    "    communities[partition[e[1]]].add_edge(e[0],e[1])\n",
    "    \n",
    "print(f\"{max_partition=}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from graphviz import Digraph\n",
    "\n",
    "\n",
    "graph_index=0\n",
    "\n",
    "total_nodes=0\n",
    "total_edges=0\n",
    "attck_nodes_set_visual=set()\n",
    "for c in communities:\n",
    "    dot = Digraph(name=\"MyPicture\", comment=\"the test\", format=\"pdf\")\n",
    "    dot.graph_attr['rankdir'] = 'LR'\n",
    "    # dot.node(name='a', label='wo', color='purple')\n",
    "    # dot.node(name='b', label='niu', color='purple')\n",
    "    # dot.node(name='c', label='che', color='purple')\n",
    "    \n",
    "    total_nodes+=len(communities[c].nodes)\n",
    "    total_edges+=len(communities[c].edges)\n",
    "    \n",
    "    subgraph_loss_sum=0\n",
    "    attack_node_count=0\n",
    "    attack_edge_count=0\n",
    "    for e in communities[c].edges:\n",
    "        try:\n",
    "            temp_edge=gg.edges[e]\n",
    "            srcnode=e['srcnode']\n",
    "            dstnode=e['dstnode']\n",
    "        except:\n",
    "            pass        \n",
    "\n",
    "        if True:\n",
    "            subgraph_loss_sum+=temp_edge['loss']\n",
    "\n",
    "    #     g.add_edge(srcnode,dstnode,srcmsg=node2msg[indexid2nodeid[srcnode]],dstmsg=node2msg[indexid2nodeid[dstnode]],loss=df['loss'][i])\n",
    "\n",
    "\n",
    "    #         dot.node( name=str(srcnode),label=str(node_index_id2msg_mal[srcnode]), color='purple',shape = 'box')\n",
    "            if \"'subject': '\" in temp_edge['srcmsg']:\n",
    "                src_shape='box'\n",
    "            elif \"'file': '\" in temp_edge['srcmsg']:\n",
    "                src_shape='oval'\n",
    "            elif \"'netflow': '\" in temp_edge['srcmsg']:\n",
    "                src_shape='diamond'\n",
    "                \n",
    "            src_shape='box'\n",
    "            if attack_edge_flag(temp_edge['srcmsg']):\n",
    "                src_node_color='red'\n",
    "                total_nodes+=1\n",
    "            else:\n",
    "                src_node_color='blue'\n",
    "                \n",
    "            src_node_color='blue'\n",
    "            \n",
    "            dot.node( name=str(hashgen(replace_path_name(temp_edge['srcmsg']))),\n",
    "                     label=str(replace_path_name(temp_edge['srcmsg'])+'\\t partition:'+str(partition[str(hashgen(replace_path_name(temp_edge['srcmsg'])))])), \n",
    "                     color=src_node_color,\n",
    "                     shape = src_shape)\n",
    "\n",
    "\n",
    "            if \"'subject': '\" in temp_edge['dstmsg']:\n",
    "                dst_shape='box'\n",
    "            elif \"'file': '\" in temp_edge['dstmsg']:\n",
    "                dst_shape='oval'\n",
    "            elif \"'netflow': '\" in temp_edge['dstmsg']:\n",
    "                dst_shape='diamond'\n",
    "\n",
    "                    \n",
    "            if \"->\" in temp_edge['dstmsg']:\n",
    "                dst_shape='diamond'\n",
    "            else:\n",
    "                dst_shape='oval'\n",
    "                \n",
    "                \n",
    "            if attack_edge_flag(temp_edge['dstmsg']):\n",
    "                dst_node_color='red'\n",
    "                total_nodes+=1\n",
    "            else:\n",
    "                dst_node_color='blue'\n",
    "            dst_node_color='blue'\n",
    "            \n",
    "            \n",
    "            dot.node( name=str(hashgen(replace_path_name(temp_edge['dstmsg']))),\n",
    "                     label=str(replace_path_name(temp_edge['dstmsg'])+'\\t partition:'+str(partition[str(hashgen(replace_path_name(temp_edge['dstmsg'])))])), \n",
    "                     color=dst_node_color,\n",
    "                     shape = dst_shape)\n",
    "\n",
    "\n",
    "    #         edgeindex=tensor_find(test_data.msg[i][16:-16],1)\n",
    "    \n",
    "            temp_edge_visual=replace_path_name(temp_edge['srcmsg'])+','+replace_path_name(temp_edge['dstmsg'])+','+str(temp_edge['time'])\n",
    "            temp_edge_hash_val=hashgen(temp_edge_visual)\n",
    "            \n",
    "            if temp_edge_hash_val in edges_attack_hashset_h051:\n",
    "                edge_color='red'\n",
    "                attck_nodes_set_visual.add(replace_path_name(temp_edge['srcmsg']))\n",
    "                attck_nodes_set_visual.add(replace_path_name(temp_edge['dstmsg']))\n",
    "                attack_edge_count+=1\n",
    "            else:\n",
    "                edge_color='blue'\n",
    "            \n",
    "#             if attack_edge_flag(temp_edge['srcmsg']) and attack_edge_flag(temp_edge['dstmsg']):\n",
    "#                 edge_color='red'\n",
    "#                 attack_edge_count+=1\n",
    "#             else:\n",
    "#                 edge_color='blue'\n",
    "                \n",
    "                \n",
    "            dot.edge(str(hashgen(replace_path_name(temp_edge['srcmsg']))),str(hashgen(replace_path_name(temp_edge['dstmsg']))), label= temp_edge['edge_type'] , color=edge_color)#+ \"  loss: \"+str(temp_edge['loss']) + \"  time: \"+str(temp_edge['time'])\n",
    "\n",
    "    #         dot.edge(str(srcnode), str(dstnode), label= temp_edge['edge_type']+ \"  loss: \"+str((temp_edge['loss'])) + \"  time: \"+str(temp_edge['time']) , color='red')\n",
    "\n",
    "\n",
    "#     if len(communities[c].edges)<2:\n",
    "#         continue\n",
    "#     if len(communities[c].edges)>1000:\n",
    "#         continue\n",
    "#         print(f\"edge num:{len(communities[c].edges)}，skip rendering\")\n",
    "        \n",
    "    print(\"Start to render the figures···\")\n",
    "    \n",
    "    dot.render('./graph_visual_h051/subgraph_'+str(graph_index), view=False)\n",
    "    print(\"subgraph loss:\",(subgraph_loss_sum/len(communities[c].edges)))\n",
    "    print(\"graph_index:\",graph_index)\n",
    "    print(\"edge_count:\",len(communities[c].edges))\n",
    "    print(\"node_count:\",len(communities[c].nodes))\n",
    "    print(f\"{attack_node_count=}\")\n",
    "    print(f\"{attack_edge_count=}\")\n",
    "    graph_index+=1\n",
    "\n",
    "print(f\"avg edges:{total_edges/len(communities)}\")\n",
    "print(f\"avg nodes:{total_nodes/len(communities)}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:pyg20]",
   "language": "python",
   "name": "conda-env-pyg20-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "236.766px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
